diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ea7732..2f34f87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,8 @@ cmake_minimum_required(VERSION 2.8.4 FATAL_ERROR) include(ExternalProject) include(CheckCXXCompilerFlag) +ADD_DEFINITIONS(-DPGXC) +ADD_DEFINITIONS(-DMADLIB) # -- Local definitions (filenames, paths, etc.) -------------------------------- @@ -70,7 +72,7 @@ if(CMAKE_COMPILER_IS_GNUCC) # http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_01 # We specify that we are POSIX.1-2001 compliant and XSI-conforming. We only # need to specify _XOPEN_SOURCE as _POSIX_C_SOURCE will be set implicitly. - set(CMAKE_C_FLAGS "-std=c99 -D_GLIBCXX_USE_CXX11_ABI=0 -pedantic -Wall -Wextra -Wno-clobbered -D_XOPEN_SOURCE=600" + set(CMAKE_C_FLAGS "-std=c99 -D_GLIBCXX_USE_CXX11_ABI=0 -fsigned-char -fpermissive -pedantic -Wall -Wextra -Wno-clobbered -D_XOPEN_SOURCE=600" CACHE STRING "Flags used by the compiler during all build types." FORCE) if((CMAKE_C_COMPILER_VERSION VERSION_EQUAL 5.0) OR (CMAKE_C_COMPILER_VERSION VERSION_GREATER 5.0)) @@ -106,7 +108,7 @@ if(CMAKE_COMPILER_IS_GNUCXX) # - long long is not part of the C++ 1998/2003 standard, but it is such a # common (and useful) extension that we do not want to hear warnings about # it. - set(CMAKE_CXX_FLAGS "-std=gnu++98 -D_GLIBCXX_USE_CXX11_ABI=0 -fdiagnostics-show-option -Wall -Wextra -pedantic -Wconversion -Wno-long-long -Wno-clobbered ${SSE_DISABLE_OPTIONS} -fstrict-aliasing" + set(CMAKE_CXX_FLAGS "-std=gnu++98 -D_GLIBCXX_USE_CXX11_ABI=0 -fsigned-char -fpermissive -fdiagnostics-show-option -Wall -Wextra -pedantic -Wconversion -Wno-long-long -Wno-clobbered ${SSE_DISABLE_OPTIONS} -fstrict-aliasing" CACHE STRING "Flags used by the compiler during all build types." FORCE) diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index 4a6e496..707ec3b 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -22,7 +22,7 @@ function(define_m4_macros OUT_M4_CMD_LINE OUT_M4_CODE) "__DBMS_VERSION_MAJOR__=${${DBMS_UC}_VERSION_MAJOR}" "__DBMS_VERSION_MINOR__=${${DBMS_UC}_VERSION_MINOR}" "__DBMS_VERSION_PATCH__=${${DBMS_UC}_VERSION_PATCH}" - "__DBMS_ARCHITECTURE__=${${DBMS_UC}_ARCHITECTURE}" + "__DBMS_ARCHITECTURE__=x86_64" "__MADLIB_VERSION__=${MADLIB_VERSION_STRING}" "__MADLIB_VERSION_MAJOR__=${MADLIB_VERSION_MAJOR}" "__MADLIB_VERSION_MINOR__=${MADLIB_VERSION_MINOR}" diff --git a/methods/array_ops/src/pg_gp/array_ops.c b/methods/array_ops/src/pg_gp/array_ops.cpp similarity index 97% rename from methods/array_ops/src/pg_gp/array_ops.c rename to methods/array_ops/src/pg_gp/array_ops.cpp index a842a60..936fd39 100644 --- a/methods/array_ops/src/pg_gp/array_ops.c +++ b/methods/array_ops/src/pg_gp/array_ops.cpp @@ -20,6 +20,46 @@ PG_MODULE_MAGIC; #endif + +#ifdef __cplusplus +extern "C" { +#endif + +Datum array_of_float(PG_FUNCTION_ARGS); +Datum array_of_bigint(PG_FUNCTION_ARGS); +Datum array_stddev(PG_FUNCTION_ARGS); +Datum array_mean(PG_FUNCTION_ARGS); +Datum array_sum_big(PG_FUNCTION_ARGS); +Datum array_sum(PG_FUNCTION_ARGS); +Datum array_abs_sum(PG_FUNCTION_ARGS); +Datum array_min(PG_FUNCTION_ARGS); +Datum array_max(PG_FUNCTION_ARGS); +Datum array_max_index(PG_FUNCTION_ARGS); +Datum array_min_index(PG_FUNCTION_ARGS); +Datum array_dot(PG_FUNCTION_ARGS); +Datum array_contains(PG_FUNCTION_ARGS); +Datum array_add(PG_FUNCTION_ARGS); +Datum array_sub(PG_FUNCTION_ARGS); +Datum array_mult(PG_FUNCTION_ARGS); +Datum array_div(PG_FUNCTION_ARGS); +Datum array_abs(PG_FUNCTION_ARGS); +Datum array_sqrt(PG_FUNCTION_ARGS); +Datum array_pow(PG_FUNCTION_ARGS); +Datum array_square(PG_FUNCTION_ARGS); +Datum array_fill_(PG_FUNCTION_ARGS); +Datum array_cos(PG_FUNCTION_ARGS); +Datum array_scalar_mult(PG_FUNCTION_ARGS); +Datum array_scalar_add(PG_FUNCTION_ARGS); +Datum array_cum_sum(PG_FUNCTION_ARGS); +Datum array_cum_prod(PG_FUNCTION_ARGS); +Datum array_filter(PG_FUNCTION_ARGS); +Datum array_normalize(PG_FUNCTION_ARGS); +Datum array_contains_null(PG_FUNCTION_ARGS); +Datum array_to_bytea(PG_FUNCTION_ARGS); + +#ifdef __cplusplus +} +#endif /* This module provides c implementations for several postgres array operations. @@ -378,7 +418,7 @@ array_of_float(PG_FUNCTION_ARGS){ errmsg("invalid array length"), errdetail("array_of_float: Size should be in [1, 1e7], %d given", size))); } - Datum* array = palloc (sizeof(Datum)*size); + Datum* array = (Datum *) palloc (sizeof(Datum)*size); for(int i = 0; i < size; ++i) { array[i] = Float8GetDatum(0); } @@ -405,7 +445,7 @@ array_of_bigint(PG_FUNCTION_ARGS){ errmsg("invalid array length"), errdetail("array_of_bigint: Size should be in [1, 1e7], %d given", size))); } - Datum* array = palloc (sizeof(Datum)*size); + Datum* array = (Datum *) palloc (sizeof(Datum)*size); for(int i = 0; i < size; ++i) { array[i] = Int64GetDatum(0); } @@ -823,9 +863,9 @@ array_square(PG_FUNCTION_ARGS){ /* * This function sets all elements to be the specified value. */ -PG_FUNCTION_INFO_V1(array_fill); +PG_FUNCTION_INFO_V1(array_fill_); Datum -array_fill(PG_FUNCTION_ARGS){ +array_fill_(PG_FUNCTION_ARGS){ if (PG_ARGISNULL(0)) { PG_RETURN_NULL(); } if (PG_ARGISNULL(1)) { PG_RETURN_NULL(); } @@ -2129,7 +2169,7 @@ array_to_bytea(PG_FUNCTION_ARGS) data_length = (nitems * TI->typlen); } - bytea *ba = palloc(VARHDRSZ + data_length); + bytea *ba = (bytea *)palloc(VARHDRSZ + data_length); SET_VARSIZE(ba, VARHDRSZ + data_length); diff --git a/methods/array_ops/src/pg_gp/array_ops.sql_in b/methods/array_ops/src/pg_gp/array_ops.sql_in index c1ec853..fc7036b 100644 --- a/methods/array_ops/src/pg_gp/array_ops.sql_in +++ b/methods/array_ops/src/pg_gp/array_ops.sql_in @@ -268,7 +268,7 @@ File array_ops.sql_in for list of functions and usage. */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_add(x anyarray, y anyarray) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_add' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -295,7 +295,7 @@ CREATE AGGREGATE MADLIB_SCHEMA.sum(/* x */ anyarray) ( */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_sub(x anyarray, y anyarray) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_sub' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -308,7 +308,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_mult(x anyarray, y anyarray) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_mult' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -321,7 +321,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_div(x anyarray, y anyarray) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_div' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -334,7 +334,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_dot(x anyarray, y anyarray) RETURNS float8 AS 'MODULE_PATHNAME', 'array_dot' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -347,7 +347,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_contains(x anyarray, y anyarray) RETURNS BOOL AS 'MODULE_PATHNAME', 'array_contains' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -359,7 +359,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_max(x anyarray) RETURNS anyelement AS 'MODULE_PATHNAME', 'array_max' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -371,7 +371,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_max_index(x anyarray) RETURNS float8[] AS 'MODULE_PATHNAME', 'array_max_index' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -383,7 +383,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_min(x anyarray) RETURNS anyelement AS 'MODULE_PATHNAME', 'array_min' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -395,7 +395,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_min_index(x anyarray) RETURNS float8[] AS 'MODULE_PATHNAME', 'array_min_index' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -407,7 +407,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_sum(x anyarray) RETURNS anyelement AS 'MODULE_PATHNAME', 'array_sum' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -420,7 +420,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_sum_big(x anyarray) RETURNS float8 AS 'MODULE_PATHNAME', 'array_sum_big' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -431,7 +431,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_abs_sum(x anyarray) RETURNS anyelement AS 'MODULE_PATHNAME', 'array_abs_sum' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -443,7 +443,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_abs(x anyarray) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_abs' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -455,7 +455,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_mean(x anyarray) RETURNS float8 AS 'MODULE_PATHNAME', 'array_mean' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -467,7 +467,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_stddev(x anyarray) RETURNS float8 AS 'MODULE_PATHNAME', 'array_stddev' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -479,7 +479,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_of_float(k integer) RETURNS float8[] AS 'MODULE_PATHNAME', 'array_of_float' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -491,7 +491,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_of_bigint(k integer) RETURNS bigint[] AS 'MODULE_PATHNAME', 'array_of_bigint' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -503,8 +503,8 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); * */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_fill(x anyarray, k anyelement) RETURNS anyarray -AS 'MODULE_PATHNAME', 'array_fill' -LANGUAGE C IMMUTABLE +AS 'MODULE_PATHNAME', 'array_fill_' +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -515,7 +515,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_cos(x anyarray) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_cos' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -527,7 +527,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_scalar_mult(x anyarray, k anyelement) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_scalar_mult' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -539,7 +539,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_scalar_add(x anyarray, k anyelement) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_scalar_add' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -551,7 +551,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); * @returns Filtered array. */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_filter(x anyarray, k anyelement, op text) RETURNS anyarray -AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE +AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -561,7 +561,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); * @returns Filtered array. */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_filter(x anyarray, k anyelement) RETURNS anyarray -AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE +AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -570,7 +570,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); * @returns Filtered array. */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_filter(x anyarray) RETURNS anyarray -AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE +AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -583,7 +583,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.array_sqrt(x anyarray) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.array_sqrt(x anyarray) RETURNS float8[] AS 'MODULE_PATHNAME', 'array_sqrt' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -597,7 +597,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.array_pow(x anyarray, y anyelement) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.array_pow(x anyarray, y anyelement) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_pow' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -610,7 +610,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.array_square(x anyarray) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.array_square(x anyarray) RETURNS float8[] AS 'MODULE_PATHNAME', 'array_square' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -622,7 +622,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ DROP FUNCTION IF EXISTS MADLIB_SCHEMA.normalize(float8[]) CASCADE; CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalize(x anyarray) RETURNS float8[] -AS 'MODULE_PATHNAME', 'array_normalize' LANGUAGE C IMMUTABLE STRICT +AS 'MODULE_PATHNAME', 'array_normalize' LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -634,7 +634,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ DROP FUNCTION IF EXISTS MADLIB_SCHEMA.array_contains_null(float8[]) CASCADE; CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_contains_null(x anyarray) RETURNS BOOLEAN -AS 'MODULE_PATHNAME', 'array_contains_null' LANGUAGE C IMMUTABLE STRICT +AS 'MODULE_PATHNAME', 'array_contains_null' LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -648,7 +648,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_cum_sum(x anyarray) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_cum_sum' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -662,7 +662,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_cum_prod(x anyarray) RETURNS anyarray AS 'MODULE_PATHNAME', 'array_cum_prod' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -741,7 +741,115 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_to_bytea(ANYARRAY) RETURNS BYTEA AS 'MODULE_PATHNAME', 'array_to_bytea' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED; !>) m4_changequote(`,') m4_ifdef(__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); +------------------------------------------------------------------ +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_sort(x ANYARRAY) RETURNS ANYARRAY AS +$BODY$ +SELECT ARRAY( + SELECT $1[s.i] AS "foo" + FROM + generate_series(array_lower($1,1), array_upper($1,1)) AS s(i) + ORDER BY foo); +$BODY$ LANGUAGE SQL IMMUTABLE; +------------------------------------------------------------------ + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.percentile_cont(myarray anyarray, percentile double precision)RETURNS double precision AS $$ +DECLARE + ary_cnt bigint; + row_num bigint; + crn double precision; + frn double precision; + calc_result double precision; + new_array double precision[]; +BEGIN + new_array := array_sort(myarray); + ary_cnt := array_length(new_array,1); + row_num := 1 + ( percentile * ( ary_cnt - 1 )); + crn := ceiling(row_num); + frn := floor(row_num); + if crn = frn and frn = row_num then + calc_result := new_array[row_num]; + else + calc_result := (crn - row_num) * new_array[frn] + (row_num - frn) * new_array[crn]; + end if; + RETURN calc_result; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +------------------------------------------------------------------ +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.percentile_disc(myarray anyarray, percentile double precision) RETURNS double precision AS $$ +DECLARE + ary_cnt bigint; + row_num double precision; + frn bigint; + calc_result double precision; + new_array double precision[]; +BEGIN + new_array := array_sort(myarray); + ary_cnt := array_length(new_array,1); + row_num := percentile * ary_cnt; + frn := ceil(row_num); + calc_result := new_array[frn]; + RETURN calc_result; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +----------------------------------------------------------------- +-- aggregation for independent var +DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(REAL[]); +CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(REAL[]) ( + SFUNC = array_cat, + PREFUNC = array_cat, + STYPE = REAL[] + + ); + +-- aggregation for dependent var +DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(SMALLINT[]); +CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(SMALLINT[]) ( + SFUNC = array_cat, + PREFUNC = array_cat, + STYPE = SMALLINT[] + + ); + +CREATE FUNCTION MADLIB_SCHEMA.convert_array_to_bytea(var REAL[]) +RETURNS BYTEA +AS +$$ +import numpy as np + +return np.array(var, dtype=np.float32).tobytes() +$$ LANGUAGE plpythonu; + +CREATE FUNCTION MADLIB_SCHEMA.convert_array_to_bytea(var SMALLINT[]) +RETURNS BYTEA +AS +$$ +import numpy as np + +return np.array(var, dtype=np.int16).tobytes() +$$ LANGUAGE plpythonu; + + +CREATE FUNCTION MADLIB_SCHEMA.convert_bytea_to_real_array(var BYTEA) +RETURNS REAL[] +AS +$$ +import numpy as np + +return np.frombuffer(var, dtype=np.float32) +$$ LANGUAGE plpythonu; + + +CREATE FUNCTION MADLIB_SCHEMA.convert_bytea_to_smallint_array(var BYTEA) +RETURNS SMALLINT[] +AS +$$ +import numpy as np + +return np.frombuffer(var, dtype=np.int16) +$$ LANGUAGE plpythonu; diff --git a/methods/kmeans/src/pg_gp/kmeans.c b/methods/kmeans/src/pg_gp/kmeans.cpp similarity index 95% rename from methods/kmeans/src/pg_gp/kmeans.c rename to methods/kmeans/src/pg_gp/kmeans.cpp index d74f65b..a4785b0 100644 --- a/methods/kmeans/src/pg_gp/kmeans.c +++ b/methods/kmeans/src/pg_gp/kmeans.cpp @@ -17,6 +17,19 @@ typedef enum { TANIMOTO } KMeansMetric; +#ifdef __cplusplus +extern "C" { +#endif +Datum internal_get_array_of_close_canopies(PG_FUNCTION_ARGS); +Datum internal_kmeans_closest_centroid(PG_FUNCTION_ARGS); +Datum internal_kmeans_agg_centroid_trans(PG_FUNCTION_ARGS); +Datum internal_kmeans_agg_centroid_merge(PG_FUNCTION_ARGS); +Datum internal_kmeans_canopy_transition(PG_FUNCTION_ARGS); +Datum internal_remove_close_canopies(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + static inline int @@ -128,7 +141,7 @@ internal_get_array_of_close_canopies(PG_FUNCTION_ARGS) get_svec_array_elms(PG_GETARG_ARRAYTYPE_P(verify_arg_nonnull(fcinfo, 1)), &all_canopies, &num_all_canopies); threshold = PG_GETARG_FLOAT8(verify_arg_nonnull(fcinfo, 2)); - metric_fn = get_metric_fn(PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 3))); + metric_fn = get_metric_fn((KMeansMetric)PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 3))); mem_context_for_function_calls = setup_mem_context_for_functional_calls(); close_canopies = (int32 *) palloc(sizeof(int32) * num_all_canopies); @@ -382,7 +395,7 @@ internal_kmeans_closest_centroid(PG_FUNCTION_ARGS) { cid = indirect ? canopy_ids[i] - ARR_LBOUND(canopy_ids_arr)[0] : i; double * centroid = c_centroids_array+cid*dimension; - MetricFunc func = get_metric_fn_for_array(dist_metric); + MetricFunc func = get_metric_fn_for_array((KMeansMetric)dist_metric); distance = (*func)(centroid, c_point_array, dimension); if (distance < min_distance) { @@ -459,7 +472,7 @@ internal_kmeans_agg_centroid_trans(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { - c_array = palloc0(expected_array_len*sizeof(float8)); + c_array = (float8 *) palloc0(expected_array_len*sizeof(float8)); rebuild_array = true; } else @@ -565,7 +578,7 @@ internal_kmeans_canopy_transition(PG_FUNCTION_ARGS) { canopies_arr = PG_GETARG_ARRAYTYPE_P(verify_arg_nonnull(fcinfo, 0)); get_svec_array_elms(canopies_arr, &canopies, &num_canopies); point = PG_GETARG_SVECTYPE_P(verify_arg_nonnull(fcinfo, 1)); - metric_fn = get_metric_fn(PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 2))); + metric_fn = get_metric_fn((KMeansMetric)PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 2))); threshold = PG_GETARG_FLOAT8(verify_arg_nonnull(fcinfo, 3)); mem_context_for_function_calls = setup_mem_context_for_functional_calls(); @@ -609,7 +622,7 @@ internal_remove_close_canopies(PG_FUNCTION_ARGS) { all_canopies_arr = PG_GETARG_ARRAYTYPE_P(verify_arg_nonnull(fcinfo, 0)); get_svec_array_elms(all_canopies_arr, &all_canopies, &num_all_canopies); - metric_fn = get_metric_fn(PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 1))); + metric_fn = get_metric_fn((KMeansMetric)PG_GETARG_INT32(verify_arg_nonnull(fcinfo, 1))); threshold = PG_GETARG_FLOAT8(verify_arg_nonnull(fcinfo, 2)); mem_context_for_function_calls = setup_mem_context_for_functional_calls(); diff --git a/methods/sketch/src/pg_gp/countmin.c b/methods/sketch/src/pg_gp/countmin.cpp similarity index 97% rename from methods/sketch/src/pg_gp/countmin.c rename to methods/sketch/src/pg_gp/countmin.cpp index 8b70ec6..152d51a 100644 --- a/methods/sketch/src/pg_gp/countmin.c +++ b/methods/sketch/src/pg_gp/countmin.cpp @@ -39,6 +39,17 @@ #include "sketch_support.h" #include "countmin.h" +#ifdef __cplusplus +extern "C" { +#endif +Datum __cmsketch_int8_trans(PG_FUNCTION_ARGS); +Datum __cmsketch_base64_final(PG_FUNCTION_ARGS); +Datum __cmsketch_merge(PG_FUNCTION_ARGS); +Datum cmsketch_dump(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + PG_FUNCTION_INFO_V1(__cmsketch_int8_trans); /* @@ -248,7 +259,7 @@ Datum __cmsketch_base64_final(PG_FUNCTION_ARGS) elog(ERROR, "invalid transition state for cmsketch"); } - out = palloc0(len); + out = (bytea *) palloc0(len); if (VARSIZE(blob) > VARHDRSZ) { sketch = (cmtransval *)VARDATA(blob); memcpy((uint8 *)VARDATA(out), sketch->sketches, len - VARHDRSZ); @@ -263,7 +274,7 @@ Datum __cmsketch_base64_final(PG_FUNCTION_ARGS) res; datalen = VARSIZE(data) - VARHDRSZ; resultlen = b64_enc_len(datalen); - result = palloc(VARHDRSZ + resultlen); + result = (text *) palloc(VARHDRSZ + resultlen); res = b64_encode(VARDATA(data), datalen, VARDATA(result)); /* Make this FATAL 'cause we've trodden on memory ... */ if (res > resultlen) diff --git a/methods/sketch/src/pg_gp/countmin.h b/methods/sketch/src/pg_gp/countmin.h index 28f1347..11476d8 100644 --- a/methods/sketch/src/pg_gp/countmin.h +++ b/methods/sketch/src/pg_gp/countmin.h @@ -172,6 +172,10 @@ int cnt_cmp_desc(const void *i, const void *j); /* UDF protos */ +#ifdef __cplusplus +extern "C" { +#endif + Datum __cmsketch_int8_trans(PG_FUNCTION_ARGS); Datum cmsketch_width_histogram(PG_FUNCTION_ARGS); Datum cmsketch_dhistogram(PG_FUNCTION_ARGS); @@ -187,5 +191,8 @@ Datum __mfvsketch_trans(PG_FUNCTION_ARGS); Datum __mfvsketch_final(PG_FUNCTION_ARGS); Datum __mfvsketch_merge(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif #endif /* _COUNTMIN_H_ */ diff --git a/methods/sketch/src/pg_gp/fm.c b/methods/sketch/src/pg_gp/fm.cpp similarity index 98% rename from methods/sketch/src/pg_gp/fm.c rename to methods/sketch/src/pg_gp/fm.cpp index 39f3413..c8b8011 100644 --- a/methods/sketch/src/pg_gp/fm.c +++ b/methods/sketch/src/pg_gp/fm.cpp @@ -104,6 +104,17 @@ typedef struct { char storage[]; } fmtransval; +#ifdef __cplusplus +extern "C" { +#endif +Datum __fmsketch_trans(PG_FUNCTION_ARGS); +Datum __fmsketch_count_distinct(PG_FUNCTION_ARGS); +Datum __fmsketch_merge(PG_FUNCTION_ARGS); +Datum big_or(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + /* check whter the contents in fmtransval::storage is safe for sortasort */ void check_sortasort(sortasort *st, size_t st_size) { size_t left_len = st_size; @@ -317,7 +328,7 @@ Datum __fmsketch_trans(PG_FUNCTION_ARGS) * zero bits for an empty FM sketch. * \param template an optional pre-existing transval whose fields we can copy in */ -bytea *fm_new(fmtransval *template) +bytea *fm_new(fmtransval *template1) { int fmsize = VARHDRSZ + sizeof(fmtransval) + FMSKETCH_SZ; /* use palloc0 to make sure it's initialized to 0 */ @@ -327,8 +338,8 @@ bytea *fm_new(fmtransval *template) SET_VARSIZE(newblob, fmsize); transval = (fmtransval *)VARDATA(newblob); /* copy over the struct values */ - if (template != NULL) - memcpy(transval, template, sizeof(fmtransval)); + if (template1 != NULL) + memcpy(transval, template1, sizeof(fmtransval)); /* set status to BIG, possibly overwriting what was in template */ transval->status = BIG; diff --git a/methods/sketch/src/pg_gp/mfvsketch.c b/methods/sketch/src/pg_gp/mfvsketch.cpp similarity index 98% rename from methods/sketch/src/pg_gp/mfvsketch.c rename to methods/sketch/src/pg_gp/mfvsketch.cpp index 57e8c5d..6192ad6 100644 --- a/methods/sketch/src/pg_gp/mfvsketch.c +++ b/methods/sketch/src/pg_gp/mfvsketch.cpp @@ -45,6 +45,16 @@ #include +#ifdef __cplusplus +extern "C" { +#endif +Datum __mfvsketch_trans(PG_FUNCTION_ARGS); +Datum __mfvsketch_final(PG_FUNCTION_ARGS); +Datum __mfvsketch_merge(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + /* check whether the content in the given bytea is safe for mfvtransval */ void check_mfvtransval(bytea *storage) { size_t left_len = VARSIZE(storage); @@ -515,7 +525,7 @@ bytea *mfvsketch_merge_c(bytea *transblob1, bytea *transblob2) { mfvtransval *transval1 = (mfvtransval *)VARDATA(transblob1); mfvtransval *transval2 = (mfvtransval *)VARDATA(transblob2); - void *newblob; + bytea *newblob; mfvtransval *newval; uint32 i, j, cnt; @@ -572,7 +582,7 @@ bytea *mfvsketch_merge_c(bytea *transblob1, bytea *transblob2) qsort(transval1->mfvs, transval1->next_mfv, sizeof(offsetcnt), cnt_cmp_desc); qsort(transval2->mfvs, transval2->next_mfv, sizeof(offsetcnt), cnt_cmp_desc); - Datum *values_seen = palloc0((2 * newval->max_mfvs) * sizeof(Datum)); + Datum *values_seen = (Datum *)palloc0((2 * newval->max_mfvs) * sizeof(Datum)); unsigned int k; /* choose top k from transval1 and transval2 */ diff --git a/methods/sketch/src/pg_gp/sketch.sql_in b/methods/sketch/src/pg_gp/sketch.sql_in index 2ae36a2..60e4ae8 100644 --- a/methods/sketch/src/pg_gp/sketch.sql_in +++ b/methods/sketch/src/pg_gp/sketch.sql_in @@ -381,28 +381,28 @@ DROP FUNCTION IF EXISTS MADLIB_SCHEMA.big_or(bitmap1 bytea, bitmap2 bytea) CASCA CREATE FUNCTION MADLIB_SCHEMA.big_or(bitmap1 bytea, bitmap2 bytea) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_trans(bitmaps bytea, input anyelement) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_trans(bitmaps bytea, input anyelement) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_count_distinct(bitmaps bytea) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_count_distinct(bitmaps bytea) RETURNS int8 AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__fmsketch_merge(bitmaps1 bytea, bitmaps2 bytea) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__fmsketch_merge(bitmaps1 bytea, bitmaps2 bytea) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fmsketch_dcount(anyelement); @@ -431,42 +431,42 @@ DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8) CASCADE CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8, int8) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8, arg2 int8) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_int8_trans(bytea, int8, int8, int8, int8) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_int8_trans(bitmaps bytea, input int8, arg1 int8, arg2 int8, arg3 int8) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_base64_final(bytea) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_base64_final(sketch bytea) RETURNS text AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__cmsketch_merge(bytea, bytea) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__cmsketch_merge(bytea, bytea) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.cmsketch(int8); @@ -580,33 +580,33 @@ DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_trans(bytea, anyelement, int4) CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_trans(bytea, anyelement, int4) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_final(bytea) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_final(bytea) RETURNS text[][] AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP FUNCTION IF EXISTS MADLIB_SCHEMA.__mfvsketch_merge(bytea, bytea) CASCADE; CREATE FUNCTION MADLIB_SCHEMA.__mfvsketch_merge(bytea, bytea) RETURNS bytea AS 'MODULE_PATHNAME' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__sketch_rightmost_one(bytea, integer, integer) -RETURNS integer AS 'MODULE_PATHNAME', 'sketch_rightmost_one' LANGUAGE C STRICT +RETURNS integer AS 'MODULE_PATHNAME', 'sketch_rightmost_one' LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__sketch_leftmost_zero(bytea, integer, integer) -RETURNS integer AS 'MODULE_PATHNAME', 'sketch_leftmost_zero' LANGUAGE C STRICT +RETURNS integer AS 'MODULE_PATHNAME', 'sketch_leftmost_zero' LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__sketch_array_set_bit_in_place(bytea, integer, integer, integer, integer) -RETURNS bytea AS 'MODULE_PATHNAME', 'sketch_array_set_bit_in_place' LANGUAGE C STRICT +RETURNS bytea AS 'MODULE_PATHNAME', 'sketch_array_set_bit_in_place' LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.mfvsketch_top_histogram( anyelement, int4); diff --git a/methods/sketch/src/pg_gp/sketch_support.c b/methods/sketch/src/pg_gp/sketch_support.cpp similarity index 98% rename from methods/sketch/src/pg_gp/sketch_support.c rename to methods/sketch/src/pg_gp/sketch_support.cpp index 1cdeb90..ca57b07 100644 --- a/methods/sketch/src/pg_gp/sketch_support.c +++ b/methods/sketch/src/pg_gp/sketch_support.cpp @@ -50,7 +50,15 @@ #include #include "sketch_support.h" - +#ifdef __cplusplus +extern "C" { +#endif +Datum sketch_array_set_bit_in_place(PG_FUNCTION_ARGS); +Datum sketch_rightmost_one(PG_FUNCTION_ARGS); +Datum sketch_leftmost_zero(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif /*! * Simple linear function to find the rightmost bit that's set to one @@ -302,7 +310,7 @@ bytea *sketch_md5_bytea(Datum dat, Oid typOid) // according to postgres' libpq/md5.c, need 33 bytes to hold // null-terminated md5 string char outbuf[MD5_HASHLEN*2+1]; - bytea *out = palloc0(MD5_HASHLEN+VARHDRSZ); + bytea *out = (bytea *) palloc0(MD5_HASHLEN+VARHDRSZ); bool byval = get_typbyval(typOid); int len = ExtractDatumLen(dat, get_typlen(typOid), byval, -1); void *datp = DatumExtractPointer(dat, byval); diff --git a/methods/sketch/src/pg_gp/sortasort.c b/methods/sketch/src/pg_gp/sortasort.cpp similarity index 99% rename from methods/sketch/src/pg_gp/sortasort.c rename to methods/sketch/src/pg_gp/sortasort.cpp index c052ca7..1bc9a72 100644 --- a/methods/sketch/src/pg_gp/sortasort.c +++ b/methods/sketch/src/pg_gp/sortasort.cpp @@ -30,6 +30,13 @@ #define SORTASORT_DATA(s) (((char *)(s->dir)) + \ (s->capacity * (sizeof (s->dir[0])))) +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __cplusplus +} +#endif + /* * get the ith item stored in sortasort */ diff --git a/methods/stemmer/src/pg_gp/porter_stemmer.c b/methods/stemmer/src/pg_gp/porter_stemmer.cpp similarity index 97% rename from methods/stemmer/src/pg_gp/porter_stemmer.c rename to methods/stemmer/src/pg_gp/porter_stemmer.cpp index 598e232..2a65862 100644 --- a/methods/stemmer/src/pg_gp/porter_stemmer.c +++ b/methods/stemmer/src/pg_gp/porter_stemmer.cpp @@ -61,6 +61,15 @@ */ #include "libstemmer/include/libstemmer.h" +#ifdef __cplusplus +extern "C" { +#endif +Datum stem_token(PG_FUNCTION_ARGS); +Datum stem_token_arr(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + static const char * stem_token_text(struct sb_stemmer *stemmer, text *token) { char * stemmed_token = TextDatumGetCString(PointerGetDatum(token)); if (strcmp(stemmed_token, "") == 0) { diff --git a/methods/stemmer/src/pg_gp/porter_stemmer.sql_in b/methods/stemmer/src/pg_gp/porter_stemmer.sql_in index 43e2805..1e39b5f 100644 --- a/methods/stemmer/src/pg_gp/porter_stemmer.sql_in +++ b/methods/stemmer/src/pg_gp/porter_stemmer.sql_in @@ -172,7 +172,7 @@ File porter_stemmer.sql_in for list of functions and usage. */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token(token text) RETURNS text AS 'MODULE_PATHNAME', 'stem_token' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -184,7 +184,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.stem_token_arr(token_arr text[]) RETURNS text[] AS 'MODULE_PATHNAME', 'stem_token_arr' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/methods/svec/src/pg_gp/SparseData.c b/methods/svec/src/pg_gp/SparseData.cpp similarity index 100% rename from methods/svec/src/pg_gp/SparseData.c rename to methods/svec/src/pg_gp/SparseData.cpp diff --git a/methods/svec/src/pg_gp/operators.c b/methods/svec/src/pg_gp/operators.cpp similarity index 93% rename from methods/svec/src/pg_gp/operators.c rename to methods/svec/src/pg_gp/operators.cpp index fb83998..4c41fd5 100644 --- a/methods/svec/src/pg_gp/operators.c +++ b/methods/svec/src/pg_gp/operators.cpp @@ -24,6 +24,65 @@ PG_MODULE_MAGIC; #endif +#ifdef __cplusplus +extern "C" { +#endif +Datum svec_concat_replicate(PG_FUNCTION_ARGS); +Datum svec_concat(PG_FUNCTION_ARGS); +Datum svec_cmp(PG_FUNCTION_ARGS); +Datum svec_ge(PG_FUNCTION_ARGS); +Datum svec_lt(PG_FUNCTION_ARGS); +Datum svec_le(PG_FUNCTION_ARGS); +Datum svec_gt(PG_FUNCTION_ARGS); +Datum svec_ne(PG_FUNCTION_ARGS); +Datum svec_eq(PG_FUNCTION_ARGS); +Datum svec_l2_cmp(PG_FUNCTION_ARGS); +Datum svec_l2_lt(PG_FUNCTION_ARGS); +Datum svec_l2_le(PG_FUNCTION_ARGS); +Datum svec_l2_eq(PG_FUNCTION_ARGS); +Datum svec_l2_ne(PG_FUNCTION_ARGS); +Datum svec_l2_gt(PG_FUNCTION_ARGS); +Datum svec_l2_ge(PG_FUNCTION_ARGS); +Datum svec_pow(PG_FUNCTION_ARGS); +Datum svec_minus(PG_FUNCTION_ARGS); +Datum svec_plus(PG_FUNCTION_ARGS); +Datum svec_mult(PG_FUNCTION_ARGS); +Datum svec_div(PG_FUNCTION_ARGS); +Datum svec_dot(PG_FUNCTION_ARGS); +Datum svec_cast_int2(PG_FUNCTION_ARGS); +Datum svec_cast_int4(PG_FUNCTION_ARGS); +Datum svec_cast_int8(PG_FUNCTION_ARGS); +Datum svec_cast_float4(PG_FUNCTION_ARGS); +Datum svec_cast_float8(PG_FUNCTION_ARGS); +Datum svec_cast_numeric(PG_FUNCTION_ARGS); +Datum float8arr_cast_int2(PG_FUNCTION_ARGS); +Datum float8arr_cast_int4(PG_FUNCTION_ARGS); +Datum float8arr_cast_int8(PG_FUNCTION_ARGS); +Datum float8arr_cast_float4(PG_FUNCTION_ARGS); +Datum float8arr_cast_float8(PG_FUNCTION_ARGS); +Datum float8arr_cast_numeric(PG_FUNCTION_ARGS); +Datum svec_cast_float8arr(PG_FUNCTION_ARGS); +Datum svec_cast_positions_float8arr(PG_FUNCTION_ARGS); +Datum float8arr_equals(PG_FUNCTION_ARGS); +Datum float8arr_dot(PG_FUNCTION_ARGS); +Datum float8arr_minus_float8arr(PG_FUNCTION_ARGS); +Datum svec_minus_float8arr(PG_FUNCTION_ARGS); +Datum float8arr_minus_svec(PG_FUNCTION_ARGS); +Datum float8arr_plus_float8arr(PG_FUNCTION_ARGS); +Datum svec_plus_float8arr(PG_FUNCTION_ARGS); +Datum float8arr_plus_svec(PG_FUNCTION_ARGS); +Datum float8arr_mult_float8arr(PG_FUNCTION_ARGS); +Datum svec_mult_float8arr(PG_FUNCTION_ARGS); +Datum float8arr_mult_svec(PG_FUNCTION_ARGS); +Datum float8arr_div_float8arr(PG_FUNCTION_ARGS); +Datum svec_div_float8arr(PG_FUNCTION_ARGS); +Datum float8arr_div_svec(PG_FUNCTION_ARGS); +Datum svec_dot_float8arr(PG_FUNCTION_ARGS); +Datum float8arr_dot_svec(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + /** * For many functions defined in this module, the operation has no meaning * if the array dimensions aren't the same, unless one of the inputs is a diff --git a/methods/svec/src/pg_gp/sparse_vector.c b/methods/svec/src/pg_gp/sparse_vector.cpp similarity index 97% rename from methods/svec/src/pg_gp/sparse_vector.c rename to methods/svec/src/pg_gp/sparse_vector.cpp index 7944419..f0177d6 100644 --- a/methods/svec/src/pg_gp/sparse_vector.c +++ b/methods/svec/src/pg_gp/sparse_vector.cpp @@ -27,6 +27,17 @@ #include "utils/memutils.h" #include "sparse_vector.h" +#ifdef __cplusplus +extern "C" { +#endif +Datum svec_send(PG_FUNCTION_ARGS); +Datum svec_recv(PG_FUNCTION_ARGS); +Datum svec_return_array(PG_FUNCTION_ARGS); +Datum svec_out(PG_FUNCTION_ARGS); +Datum svec_in(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif /** * @return An array of float8s obtained by converting a given sparse vector */ diff --git a/methods/svec/src/pg_gp/sparse_vector.h b/methods/svec/src/pg_gp/sparse_vector.h index e29185b..18a5171 100644 --- a/methods/svec/src/pg_gp/sparse_vector.h +++ b/methods/svec/src/pg_gp/sparse_vector.h @@ -104,6 +104,10 @@ void check_dimension(SvecType *svec1, SvecType *svec2, char *msg); double svec_svec_dot_product(SvecType *svec1, SvecType *svec2); SparseData sdata_uncompressed_from_float8arr_internal(ArrayType *array); +#ifdef __cplusplus +extern "C" { +#endif + Datum svec_in(PG_FUNCTION_ARGS); Datum svec_out(PG_FUNCTION_ARGS); Datum svec_return_vector(PG_FUNCTION_ARGS); @@ -154,4 +158,8 @@ Datum float8arr_cast_numeric(PG_FUNCTION_ARGS); Datum svec_cast_float8arr(PG_FUNCTION_ARGS); Datum svec_cast_positions_float8arr(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + #endif /* SPARSEVECTOR_H */ diff --git a/methods/svec/src/pg_gp/svec.sql_in b/methods/svec/src/pg_gp/svec.sql_in index 2d0e21e..4f484c0 100644 --- a/methods/svec/src/pg_gp/svec.sql_in +++ b/methods/svec/src/pg_gp/svec.sql_in @@ -634,7 +634,7 @@ CREATE TYPE MADLIB_SCHEMA.svec; CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_in(cstring) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME' - LANGUAGE C IMMUTABLE STRICT + LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Converts SVEC to CSTRING. @@ -642,7 +642,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_in(cstring) CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_out(MADLIB_SCHEMA.svec) RETURNS cstring AS 'MODULE_PATHNAME' - LANGUAGE C IMMUTABLE STRICT + LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Converts SVEC internal representation to SVEC. @@ -650,7 +650,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_out(MADLIB_SCHEMA.svec) CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_recv(internal) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME' - LANGUAGE C IMMUTABLE STRICT + LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Converts SVEC to BYTEA. @@ -658,7 +658,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_recv(internal) CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_send(MADLIB_SCHEMA.svec) RETURNS bytea AS 'MODULE_PATHNAME' - LANGUAGE C IMMUTABLE STRICT + LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE TYPE MADLIB_SCHEMA.svec ( @@ -673,232 +673,232 @@ CREATE TYPE MADLIB_SCHEMA.svec ( --! Adds two SVECs together, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_plus(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_plus' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_plus(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_plus' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Minus second SVEC from the first, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_minus(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_minus' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_minus(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_minus' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Divides the first SVEC by the second, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_div(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_div' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_div(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_div' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Multiplies two SVEVs together, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mult(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_mult' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mult(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_mult' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Raises each element of the first SVEC to the power given by second SVEC, which must have dimension 1 (a scalar). --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_pow(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_pow' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_pow(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_pow' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Compare two SVECs, -1 for left side smaller, 1 for greater, 0 for equal --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cmp(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_cmp' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cmp(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_cmp' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the left SVEC is greater or equal to the right. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_ge(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_ge' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_ge(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_ge' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the left SVEC is less than the right. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_lt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_lt' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_lt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_lt' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the left SVEC is less or equal to the right. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_le(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_le' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_le(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_le' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the left SVEC is greater. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_gt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_gt' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_gt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_gt' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if two SVECs are not equal. If the two SVECs are of different size, then will return true. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_ne(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_ne' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_ne(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_ne' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if two SVECs are equal. If the two SVECs are of different size, then will return false. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_eq(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_eq' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_eq(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS boolean AS 'MODULE_PATHNAME', 'svec_eq' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if two float8 arrays are equal --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_eq(float8[],float8[]) RETURNS boolean AS 'MODULE_PATHNAME', 'float8arr_equals' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_eq(float8[],float8[]) RETURNS boolean AS 'MODULE_PATHNAME', 'float8arr_equals' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Minus second array from the first array, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_minus_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_minus_float8arr' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_minus_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_minus_float8arr' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Minus second SVEC from the first array, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_minus_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_minus_svec' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_minus_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_minus_svec' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Minus second array from the first SVEC, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_minus_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_minus_float8arr' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_minus_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_minus_float8arr' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Adds two arrays together, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_plus_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_plus_float8arr' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_plus_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_plus_float8arr' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Adds an array and an SVEC, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_plus_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_plus_svec' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_plus_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_plus_svec' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Adds an SVEC and an array, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_plus_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_plus_float8arr' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_plus_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_plus_float8arr' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Multiplies two float8 arrays, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_mult_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_mult_float8arr' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_mult_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_mult_float8arr' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Multiplies an array and an SVEC, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_mult_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_mult_svec' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_mult_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_mult_svec' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Multiplies an SVEC and an array, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mult_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_mult_float8arr' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_mult_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_mult_float8arr' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Divides a float8 array by another, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_div_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_div_float8arr' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_div_float8arr(float8[],float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_div_float8arr' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Divides a float8 array by an SVEC, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_div_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_div_svec' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_div_svec(float8[],MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'float8arr_div_svec' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Divides an SVEC by a float8 array, element by element. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_div_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_div_float8arr' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_div_float8arr(MADLIB_SCHEMA.svec,float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_div_float8arr' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Computes the dot product of two SVECs. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_dot' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_dot' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Computes the dot product of two float8 arrays. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(float8[],float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_dot' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(float8[],float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_dot' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Computes the dot product of an SVEC and a float8 array. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(MADLIB_SCHEMA.svec,float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_dot_float8arr' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(MADLIB_SCHEMA.svec,float8[]) RETURNS float8 AS 'MODULE_PATHNAME', 'svec_dot_float8arr' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Computes the dot product of a float8 array and an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(float8[],MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_dot_svec' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_dot(float8[],MADLIB_SCHEMA.svec) RETURNS float8 AS 'MODULE_PATHNAME', 'float8arr_dot_svec' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts an int2 into an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int2(int2) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int2' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int2(int2) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int2' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts an int4 into an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int4(int4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int4' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int4(int4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int4' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts an int8 into an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int8(bigint) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int8' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_int8(bigint) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_int8' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts a float4 into an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float4(float4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float4' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float4(float4) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float4' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts a float8 into an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float8(float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float8' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float8(float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float8' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts a numeric into an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_numeric(numeric) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_numeric' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_numeric(numeric) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_numeric' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts an int2 into a float8 array. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int2(int2) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int2' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int2(int2) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int2' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts an int4 into a float8 array. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int4(int4) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int4' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int4(int4) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int4' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts an int8 into a float8 array. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int8(bigint) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int8' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_int8(bigint) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_int8' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts a float4 into a float8 array. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_float4(float4) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_float4' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_float4(float4) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_float4' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts a float8 into a float8 array. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_float8(float8) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_float8' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_float8(float8) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_float8' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts a numeric into a float8 array. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_numeric(numeric) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_numeric' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.float8arr_cast_numeric(numeric) RETURNS float8[] AS 'MODULE_PATHNAME', 'float8arr_cast_numeric' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts a float8 into an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float8arr(float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float8arr' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_float8arr(float8[]) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_float8arr' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts an array of int8 positions, float8 values into an SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_positions_float8arr(int8[],float8[],int8,float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_positions_float8arr' STRICT LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_cast_positions_float8arr(int8[],float8[],int8,float8) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_cast_positions_float8arr' STRICT LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Casts an SVEC into a float8 array. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_return_array(MADLIB_SCHEMA.svec) RETURNS float8[] AS 'MODULE_PATHNAME', 'svec_return_array' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_return_array(MADLIB_SCHEMA.svec) RETURNS float8[] AS 'MODULE_PATHNAME', 'svec_return_array' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Concatenates two SVECs. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_concat(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_concat' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_concat(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_concat' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Replicates n copies of an SVEC and concatenates them together. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_concat_replicate(int4,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_concat_replicate' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_concat_replicate(int4,MADLIB_SCHEMA.svec) RETURNS MADLIB_SCHEMA.svec AS 'MODULE_PATHNAME', 'svec_concat_replicate' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /* @@ -1108,37 +1108,37 @@ CREATE OPERATOR MADLIB_SCHEMA.= ( -- Comparisons based on L2 Norm --! Returns true if the l2 norm of the first SVEC is less than that of the second SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_lt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_lt' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_lt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_lt' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the l2 norm of the first SVEC is less than or equal to that of the second SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_le(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_le' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_le(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_le' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the l2 norm of the first SVEC is equal to that of the second SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_eq(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_eq' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_eq(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_eq' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the l2 norm of the first SVEC is not equal to that of the second SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_ne(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_ne' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_ne(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_ne' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the l2 norm of the first SVEC is greater than that of the second SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_gt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_gt' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_gt(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_gt' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns true if the l2 norm of the first SVEC is greater than or equal to that of the second SVEC. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_ge(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_ge' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_ge(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS bool AS 'MODULE_PATHNAME', 'svec_l2_ge' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --! Returns a value indicating the relative values of the l2 norms of two SVECs. --! -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_cmp(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_l2_cmp' LANGUAGE C IMMUTABLE +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.svec_l2_cmp(MADLIB_SCHEMA.svec,MADLIB_SCHEMA.svec) RETURNS integer AS 'MODULE_PATHNAME', 'svec_l2_cmp' LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /* diff --git a/methods/svec_util/src/pg_gp/generate_svec.c b/methods/svec_util/src/pg_gp/generate_svec.cpp similarity index 95% rename from methods/svec_util/src/pg_gp/generate_svec.c rename to methods/svec_util/src/pg_gp/generate_svec.cpp index 1d7012c..b081b82 100644 --- a/methods/svec_util/src/pg_gp/generate_svec.c +++ b/methods/svec_util/src/pg_gp/generate_svec.cpp @@ -9,6 +9,14 @@ #include "../../../svec/src/pg_gp/sparse_vector.h" +#ifdef __cplusplus +extern "C" { +#endif +Datum generate_sparse_vector(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + PG_FUNCTION_INFO_V1(generate_sparse_vector); Datum generate_sparse_vector(PG_FUNCTION_ARGS) { diff --git a/methods/svec_util/src/pg_gp/gp_sfv.c b/methods/svec_util/src/pg_gp/gp_sfv.cpp similarity index 99% rename from methods/svec_util/src/pg_gp/gp_sfv.c rename to methods/svec_util/src/pg_gp/gp_sfv.cpp index 7b2aeb6..1cbdb1b 100644 --- a/methods/svec_util/src/pg_gp/gp_sfv.c +++ b/methods/svec_util/src/pg_gp/gp_sfv.cpp @@ -17,7 +17,13 @@ #include "../../../svec/src/pg_gp/sparse_vector.h" +#ifdef __cplusplus +extern "C" { +#endif Datum gp_extract_feature_histogram(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif static void gp_extract_feature_histogram_errout(char *msg); diff --git a/methods/svec_util/src/pg_gp/svec_util.c b/methods/svec_util/src/pg_gp/svec_util.cpp similarity index 95% rename from methods/svec_util/src/pg_gp/svec_util.c rename to methods/svec_util/src/pg_gp/svec_util.cpp index 12c839c..23e43d8 100644 --- a/methods/svec_util/src/pg_gp/svec_util.c +++ b/methods/svec_util/src/pg_gp/svec_util.cpp @@ -24,11 +24,54 @@ PG_MODULE_MAGIC; #endif +#ifdef __cplusplus +extern "C" { +#endif +Datum float8_min(PG_FUNCTION_ARGS); +Datum float8_max(PG_FUNCTION_ARGS); +Datum svec_to_string(PG_FUNCTION_ARGS); +Datum svec_from_string(PG_FUNCTION_ARGS); +Datum svec_unnest(PG_FUNCTION_ARGS); +Datum svec_dimension(PG_FUNCTION_ARGS); +Datum svec_lapply(PG_FUNCTION_ARGS); +Datum svec_append(PG_FUNCTION_ARGS); +Datum svec_proj(PG_FUNCTION_ARGS); +Datum svec_subvec(PG_FUNCTION_ARGS); +Datum svec_reverse(PG_FUNCTION_ARGS); +Datum svec_change(PG_FUNCTION_ARGS); +Datum svec_eq_non_zero(PG_FUNCTION_ARGS); +Datum svec_contains(PG_FUNCTION_ARGS); +Datum svec_count(PG_FUNCTION_ARGS); +Datum svec_l2norm(PG_FUNCTION_ARGS); +Datum svec_svec_l2norm(PG_FUNCTION_ARGS); +Datum svec_svec_l1norm(PG_FUNCTION_ARGS); +Datum svec_svec_angle(PG_FUNCTION_ARGS); +Datum svec_svec_tanimoto_distance(PG_FUNCTION_ARGS); +Datum svec_normalize(PG_FUNCTION_ARGS); +Datum svec_l1norm(PG_FUNCTION_ARGS); +Datum svec_summate(PG_FUNCTION_ARGS); +Datum svec_log(PG_FUNCTION_ARGS); +Datum float8arr_l1norm(PG_FUNCTION_ARGS); +Datum float8arr_summate(PG_FUNCTION_ARGS); +Datum float8arr_l2norm(PG_FUNCTION_ARGS); +Datum float8arr_hash(PG_FUNCTION_ARGS); +Datum svec_pivot(PG_FUNCTION_ARGS); +Datum float8arr_median(PG_FUNCTION_ARGS); +Datum svec_median(PG_FUNCTION_ARGS); +Datum svec_nonbase_positions(PG_FUNCTION_ARGS); +Datum svec_nonbase_values(PG_FUNCTION_ARGS); +Datum svec_hash(PG_FUNCTION_ARGS); +Datum svec_mean_transition(PG_FUNCTION_ARGS); +Datum svec_mean_prefunc(PG_FUNCTION_ARGS); +Datum svec_mean_final(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + /* * Basic floating point operators like MIN,MAX */ PG_FUNCTION_INFO_V1( float8_min ); -Datum float8_min(PG_FUNCTION_ARGS); Datum float8_min(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { @@ -137,7 +180,7 @@ Datum svec_unnest(PG_FUNCTION_ARGS) /* stuff done on every call of the function */ funcctx = SRF_PERCALL_SETUP(); - fctx = funcctx->user_fctx; + fctx = (svec_unnest_fctx *) funcctx->user_fctx; run_length = compword_to_int8(fctx->index_position); @@ -573,7 +616,7 @@ Datum svec_normalize(PG_FUNCTION_ARGS) norm = l2norm_sdata_values_double(sdata); - op_sdata_by_scalar_inplace( 3, (char *)&norm, sdata, 2); + op_sdata_by_scalar_inplace( (operation_t)3, (char *)&norm, sdata, 2); PG_RETURN_SVECTYPE_P( svec_from_sparsedata( sdata, true)); } @@ -915,7 +958,7 @@ partition_select (char **lists, size_t nlists, size_t *widths, tmp = (char **)palloc(nlists*sizeof(char *)); for (uint64 i=0;i +#ifdef __cplusplus +extern "C" { +#endif +Datum exec_sql_using(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + // SearchSysCache1 first appeared with commit e26c539 by Robert Haas // on Sun, 14 Feb 2010 18:42:19 +0000. If this macro is // not defined, we assume to be on a version of PostgreSQL that expects five @@ -105,7 +113,7 @@ exec_sql_using(PG_FUNCTION_ARGS) { for (int i = 1; i < nargs; i++){ if (PG_ARGISNULL(i)) { if (nulls == NULL) { - nulls = palloc0(sizeof(char) * (nargs - 1)); + nulls = (char *) palloc0(sizeof(char) * (nargs - 1)); memset(nulls, ' ', nargs - 1); } nulls[i - 1] = 'n'; diff --git a/methods/utils/src/pg_gp/noop.c b/methods/utils/src/pg_gp/noop.cpp similarity index 59% rename from methods/utils/src/pg_gp/noop.c rename to methods/utils/src/pg_gp/noop.cpp index 5521cad..7ac77cd 100644 --- a/methods/utils/src/pg_gp/noop.c +++ b/methods/utils/src/pg_gp/noop.cpp @@ -1,6 +1,14 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif +Datum noop(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + PG_FUNCTION_INFO_V1(noop); Datum noop(PG_FUNCTION_ARGS) { diff --git a/methods/utils/src/pg_gp/to_char.c b/methods/utils/src/pg_gp/to_char.cpp similarity index 89% rename from methods/utils/src/pg_gp/to_char.c rename to methods/utils/src/pg_gp/to_char.cpp index 2d33676..2dc2f43 100644 --- a/methods/utils/src/pg_gp/to_char.c +++ b/methods/utils/src/pg_gp/to_char.cpp @@ -7,6 +7,14 @@ PG_MODULE_MAGIC; #endif +#ifdef __cplusplus +extern "C" { +#endif +Datum __to_text(PG_FUNCTION_ARGS); +#ifdef __cplusplus +} +#endif + PG_FUNCTION_INFO_V1(__to_text); /* * @brief Cast a value to text. On some databases, there diff --git a/pom.xml b/pom.xml index 4d0eec1..d487d18 100644 --- a/pom.xml +++ b/pom.xml @@ -122,20 +122,20 @@ examples/gpce/sketches.sql examples/gpce/svd-mf.sql examples/gpce/svm.sql - methods/array_ops/src/pg_gp/array_ops.c + methods/array_ops/src/pg_gp/array_ops.cpp methods/array_ops/src/pg_gp/array_ops.sql_in methods/array_ops/src/pg_gp/test/array_ops.sql_in - methods/kmeans/src/pg_gp/kmeans.c + methods/kmeans/src/pg_gp/kmeans.cpp methods/sketch/src/pg_gp/__init__.py_in - methods/sketch/src/pg_gp/countmin.c + methods/sketch/src/pg_gp/countmin.cpp methods/sketch/src/pg_gp/countmin.h methods/sketch/src/pg_gp/countmin.py_in methods/sketch/src/pg_gp/fm.c - methods/sketch/src/pg_gp/mfvsketch.c + methods/sketch/src/pg_gp/mfvsketch.cpp methods/sketch/src/pg_gp/sketch.sql_in - methods/sketch/src/pg_gp/sketch_support.c + methods/sketch/src/pg_gp/sketch_support.cpp methods/sketch/src/pg_gp/sketch_support.h - methods/sketch/src/pg_gp/sortasort.c + methods/sketch/src/pg_gp/sortasort.cpp methods/sketch/src/pg_gp/sortasort.h methods/sketch/src/pg_gp/sql/cm.sql_in methods/sketch/src/pg_gp/sql/fm.sql_in @@ -143,25 +143,25 @@ methods/sketch/src/pg_gp/sql/support.sql_in methods/sketch/src/pg_gp/uncrustify.cfg methods/svec/src/pg_gp/float_specials.h - methods/svec/src/pg_gp/operators.c - methods/svec/src/pg_gp/sparse_vector.c + methods/svec/src/pg_gp/operators.cpp + methods/svec/src/pg_gp/sparse_vector.cpp methods/svec/src/pg_gp/sparse_vector.h - methods/svec/src/pg_gp/SparseData.c + methods/svec/src/pg_gp/SparseData.cpp methods/svec/src/pg_gp/SparseData.h methods/svec/src/pg_gp/svec.sql_in methods/svec_util/src/pg_gp/__init__.py_in - methods/svec_util/src/pg_gp/generate_svec.c + methods/svec_util/src/pg_gp/generate_svec.cpp methods/svec_util/src/pg_gp/generate_svec.py_in - methods/svec_util/src/pg_gp/gp_sfv.c + methods/svec_util/src/pg_gp/gp_sfv.cpp methods/svec_util/src/pg_gp/sql/gp_sfv.sql_in.todo.sql methods/svec_util/src/pg_gp/sql/gp_sfv_sort_order.sql_in methods/svec_util/src/pg_gp/sql/svec_test.sql_in - methods/svec_util/src/pg_gp/svec_util.c + methods/svec_util/src/pg_gp/svec_util.cpp methods/svec_util/src/pg_gp/svec_util.h methods/svec_util/src/pg_gp/svec_util.sql_in - methods/utils/src/pg_gp/exec_sql_using.c - methods/utils/src/pg_gp/noop.c - methods/utils/src/pg_gp/to_char.c + methods/utils/src/pg_gp/exec_sql_using.cpp + methods/utils/src/pg_gp/noop.cpp + methods/utils/src/pg_gp/to_char.cpp src/bin/CMakeLists.txt src/bin/madpack src/CMakeLists.txt diff --git a/src/config/Modules.yml b/src/config/Modules.yml index f2da10f..c1e5004 100644 --- a/src/config/Modules.yml +++ b/src/config/Modules.yml @@ -11,8 +11,8 @@ modules: - name: convex depends: ['utilities'] - name: crf - - name: deep_learning - depends: ['utilities'] +# - name: deep_learning +# depends: ['utilities'] - name: elastic_net - name: glm depends: ['utilities'] @@ -29,7 +29,7 @@ modules: depends: ['utilities'] - name: pca depends: ['linalg', 'convex'] - - name: pmml +# - name: pmml - name: prob - name: recursive_partitioning depends: ['utilities'] @@ -52,3 +52,9 @@ modules: - name: validation depends: ['array_ops', 'regress'] - name: stemmer + - name: xgboost_gs + depends: ['utilities'] + - name: facebook_prophet + depends: ['utilities'] + - name: gbdt + depends: ['utilities', 'recursive_partitioning'] diff --git a/src/config/Ports.yml b/src/config/Ports.yml index 78327d7..6502c96 100644 --- a/src/config/Ports.yml +++ b/src/config/Ports.yml @@ -9,3 +9,6 @@ postgres: greenplum: name: Greenplum DB + +opengauss: + name: openGauss DB diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py index 2af0480..68a6566 100644 --- a/src/madpack/madpack.py +++ b/src/madpack/madpack.py @@ -24,7 +24,7 @@ from utilities import is_rev_gte from utilities import remove_comments_from_sql from utilities import run_query # Required Python version -py_min_ver = [2, 6] +py_min_ver = [2, 7] # Check python version if sys.version_info[:2] < py_min_ver: @@ -59,7 +59,7 @@ portid_list = [] for port in ports: portid_list.append(port) -SUPPORTED_PORTS = ('postgres', 'greenplum') +SUPPORTED_PORTS = ('postgres', 'greenplum', 'opengauss') # Global variables portid = None # Target port ID (eg: pg90, gp40) @@ -110,7 +110,7 @@ def _get_relative_maddir(maddir, port): madlib function definition. Replacing the GPHOME path with the equivalent relative path makes it simpler to perform DB upgrades without breaking MADlib. """ - if port == 'postgres': + if port == 'postgres' or port == 'opengauss': # do nothing for postgres return maddir @@ -167,18 +167,22 @@ def _run_m4_and_append(schema, maddir_mod_py, module, sqlfile, if pre_sql: output_filehandle.writelines([pre_sql, '\n\n']) # Find the madpack dir (platform specific or generic) + global portid + portid_old = portid + portid = 'postgres' if portid == 'opengauss' else portid if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/madpack"): maddir_madpack = maddir + "/ports/" + portid + "/" + dbver + "/madpack" else: maddir_madpack = maddir + "/madpack" maddir_ext_py = maddir + "/lib/python" + portid = portid_old m4args = ['m4', '-P', '-DMADLIB_SCHEMA=' + schema, '-DPLPYTHON_LIBDIR=' + maddir_mod_py, '-DEXT_PYTHON_LIBDIR=' + maddir_ext_py, - '-DMODULE_PATHNAME=' + maddir_lib, + '-DMODULE_PATHNAME=' + '$libdir/libmadlib.so', '-DMODULE_NAME=' + module, '-I' + maddir_madpack, sqlfile] @@ -213,7 +217,7 @@ def _run_install_check_sql(schema, maddir_mod_py, module, sqlfile, sub_module = '' # Run the SQL using DB command-line utility if portid in SUPPORTED_PORTS: - sqlcmd = 'psql' + sqlcmd = 'gsql' # Test the DB cmd line utility std, err = subprocess.Popen(['which', sqlcmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() @@ -226,12 +230,8 @@ def _run_install_check_sql(schema, maddir_mod_py, module, sqlfile, '-p', con_args['host'].split(':')[1], '-d', con_args['database'], '-U', con_args['user'], - '--no-password', + '-W', con_args['password'], '-f', tmpfile] - runenv = os.environ - if 'password' in con_args: - runenv["PGPASSWORD"] = con_args['password'] - runenv["PGOPTIONS"] = '-c client_min_messages=notice' # Open log file try: @@ -243,7 +243,7 @@ def _run_install_check_sql(schema, maddir_mod_py, module, sqlfile, # Run the SQL try: info_(this, "> ... executing " + tmpfile, verbose) - retval = subprocess.call(runcmd, env=runenv, stdout=log, stderr=log) + retval = subprocess.call(runcmd, stdout=log, stderr=log) except: error_(this, "Failed executing %s" % tmpfile, False) raise Exception @@ -261,7 +261,7 @@ def _run_sql_file(schema, sqlfile): """ # Run the SQL using DB command-line utility if portid in SUPPORTED_PORTS: - sqlcmd = 'psql' + sqlcmd = 'gsql' # Test the DB cmd line utility std, err = subprocess.Popen(['which', sqlcmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() @@ -274,13 +274,9 @@ def _run_sql_file(schema, sqlfile): '-p', con_args['host'].split(':')[1], '-d', con_args['database'], '-U', con_args['user'], - '--no-password', - '--single-transaction', + '-W', con_args['password'], + #'--single-transaction', '-f', sqlfile] - runenv = os.environ - if 'password' in con_args: - runenv["PGPASSWORD"] = con_args['password'] - runenv["PGOPTIONS"] = '-c client_min_messages=notice' # Open log file logfile = sqlfile + '.log' @@ -294,7 +290,7 @@ def _run_sql_file(schema, sqlfile): try: info_(this, "> ... executing " + sqlfile, verbose) info_(this, ' '.join(runcmd), verbose) - retval = subprocess.call(runcmd, env=runenv, stdout=log, stderr=log) + retval = subprocess.call(runcmd, stdout=log, stderr=log) except: error_(this, "Failed executing %s" % sqlfile, False) raise Exception @@ -353,6 +349,12 @@ def _check_db_port(portid): return True elif portid == 'greenplum': return True + if row and row[0]['version'].lower().find('opengauss') >= 0: + if portid == 'opengauss': + return True + if row and row[0]['version'].find('GaussDB') >= 0: + if portid == 'opengauss': + return True return False # ------------------------------------------------------------------------------ @@ -469,6 +471,8 @@ def _db_upgrade(schema, filehandle, db_madlib_ver): info_(this, "\tDetecting dependencies...", True) info_(this, "\tLoading change list...", True) + portid_old = portid + portid = 'postgres' if portid == 'opengauss' else portid ch = uu.ChangeHandler(schema, portid, con_args, maddir, db_madlib_ver, filehandle) info_(this, "\tDetecting table dependencies...", True) @@ -476,7 +480,7 @@ def _db_upgrade(schema, filehandle, db_madlib_ver): info_(this, "\tDetecting view dependencies...", True) vd = uu.ViewDependency(schema, portid, con_args) - + portid = portid_old abort = False if td.has_dependency(): info_(this, "*" * 50, True) @@ -562,7 +566,10 @@ def _db_upgrade(schema, filehandle, db_madlib_ver): info_(this, "No dependency problem found, continuing to upgrade ...", True) info_(this, "\tReading existing UDAs/UDTs...", False) + portid_old = portid + portid = 'postgres' if portid == 'opengauss' else portid sc = uu.ScriptCleaner(schema, portid, con_args, ch) + portid = portid_old info_(this, "Script Cleaner initialized ...", False) ch.drop_changed_uda() @@ -632,6 +639,7 @@ def _process_py_sql_files_in_modules(modset, args_dict): error_(this, "Invalid madpack operation: %s" % madpack_cmd, True) # Perform operations on all modules + portid_new = 'postgres' if portid == 'opengauss' else portid for moduleinfo in portspecs['modules']: # Get the module name module = moduleinfo['name'] @@ -641,14 +649,14 @@ def _process_py_sql_files_in_modules(modset, args_dict): continue # Find the Python module dir (platform specific or generic) - if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/modules/" + module): - maddir_mod_py = maddir + "/ports/" + portid + "/" + dbver + "/modules" + if os.path.isdir(maddir + "/ports/" + portid_new + "/" + dbver + "/modules/" + module): + maddir_mod_py = maddir + "/ports/" + portid_new + "/" + dbver + "/modules" else: maddir_mod_py = maddir + "/modules" # Find the SQL module dir (platform specific or generic) - if os.path.isdir(maddir + "/ports/" + portid + "/modules/" + module): - maddir_mod_sql = maddir + "/ports/" + portid + "/modules" + if os.path.isdir(maddir + "/ports/" + portid_new + "/modules/" + module): + maddir_mod_sql = maddir + "/ports/" + portid_new + "/modules" elif os.path.isdir(maddir + "/modules/" + module): maddir_mod_sql = maddir + "/modules" else: @@ -757,8 +765,7 @@ def _execute_per_module_unit_test_algo(module, pyfile, cur_tmpdir): run_start = datetime.datetime.now() # Run the python unit test file runcmd = ["python", pyfile] - runenv = os.environ - retval = subprocess.call(runcmd, env=runenv, stdout=log, stderr=log) + retval = subprocess.call(runcmd, stdout=log, stderr=log) run_end = datetime.datetime.now() milliseconds = round((run_end - run_start).seconds * 1000 + (run_end - run_start).microseconds / 1000) @@ -783,15 +790,16 @@ def _execute_per_module_install_dev_check_algo(schema, test_user, test_schema = "madlib_installcheck_%s" % (module) _internal_run_query("DROP SCHEMA IF EXISTS %s CASCADE; CREATE SCHEMA %s;" % (test_schema, test_schema), True) - _internal_run_query("GRANT ALL ON SCHEMA %s TO \"%s\";" % + _internal_run_query("GRANT ALL ON SCHEMA %s TO %s;" % (test_schema, test_user), True) # Switch to test user and prepare the search_path pre_sql = '-- Switch to test user:\n' \ - 'SET ROLE \"%s\";\n' \ + "SET ROLE %s password '%s';\n" \ '-- Set SEARCH_PATH for install-check:\n' \ 'SET search_path=%s,%s;\n' \ - % (test_user, test_schema, schema) + 'SET behavior_compat_options = \'bind_procedure_searchpath\';\n' \ + % (test_user, con_args['password'], test_schema, schema) # Set file names @@ -1031,9 +1039,10 @@ def run_install_check(args, testcase, madpack_cmd): _internal_run_query("DROP OWNED BY \"%s\" CASCADE;" % (test_user), True) _internal_run_query("DROP USER IF EXISTS \"%s\";" % (test_user), True) - _internal_run_query("CREATE USER \"%s\" WITH SUPERUSER NOINHERIT;" % (test_user), True) - _internal_run_query("GRANT USAGE ON SCHEMA %s TO \"%s\";" % (schema, test_user), True) - _internal_run_query("GRANT ALL PRIVILEGES ON DATABASE \"%s\" TO \"%s\";" % (db_name, test_user), True) + _internal_run_query("CREATE USER %s WITH SYSADMIN NOINHERIT password '%s';" % (test_user, con_args['password']), True) + + _internal_run_query("GRANT USAGE ON SCHEMA %s TO %s;" % (schema, test_user), True) + _internal_run_query("GRANT ALL PRIVILEGES ON DATABASE %s TO %s;" % (db_name, test_user), True) # 2) Run test SQLs info_(this, "> Running %s scripts for:" % madpack_cmd, verbose) @@ -1262,8 +1271,7 @@ def main(argv): con_args['host'] = c_host + ':' + c_port con_args['database'] = c_db con_args['user'] = c_user - if c_pass is not None: - con_args['password'] = c_pass + con_args['password'] = getpass.getpass("Password for user %s: " % c_user) # Try connecting to the database info_(this, "Testing database connection...", verbose) @@ -1288,7 +1296,7 @@ def main(argv): # Get MADlib version in DB db_madlib_ver = get_db_madlib_version(con_args, schema) - portdir = os.path.join(maddir, "ports", portid) + portdir = os.path.join(maddir, "ports", 'postgres' if portid == 'opengauss' else portid) supportedVersions = [dirItem for dirItem in os.listdir(portdir) if os.path.isdir(os.path.join(portdir, dirItem)) and re.match("^\d+", dirItem)] @@ -1319,7 +1327,7 @@ def main(argv): else: # only need the first two digits for <= 4.3.4 dbver = '.'.join(map(str, dbver_split[:2])) - elif portid == 'postgres': + elif portid == 'postgres' or portid == 'opengauss': if is_rev_gte(dbver_split, get_rev_num('10.0')): # Postgres starting 10.0 uses semantic versioning. Hence, # only need first digit for major version. @@ -1335,6 +1343,8 @@ def main(argv): # Adjust MADlib directories for this port (if they exist) global maddir_conf + portid_old = portid + portid = 'postgres' if portid == 'opengauss' else portid if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/config"): maddir_conf = maddir + "/ports/" + portid + "/" + dbver + "/config" else: @@ -1347,7 +1357,7 @@ def main(argv): "/lib/libmadlib.so" else: maddir_lib = maddir + "/lib/libmadlib.so" - + portid = portid_old # Get the list of modules for this port global portspecs portspecs = configyml.get_modules(maddir_conf) diff --git a/src/madpack/utilities.py b/src/madpack/utilities.py index 9e4e95b..304f7a3 100644 --- a/src/madpack/utilities.py +++ b/src/madpack/utilities.py @@ -83,7 +83,7 @@ def remove_comments_from_sql(sql): def run_query(sql, con_args, show_error=True): # Define sqlcmd - sqlcmd = 'psql' + sqlcmd = 'gsql' delimiter = ' <$madlib_delimiter$> ' # Test the DB cmd line utility @@ -98,10 +98,7 @@ def run_query(sql, con_args, show_error=True): '-p', con_args['host'].split(':')[1], '-d', con_args['database'], '-U', con_args['user'], - '-F', delimiter, - '--no-password', - '--no-psqlrc', - '--no-align', + '-W', con_args['password'], '-c', sql] runenv = os.environ if 'password' in con_args: @@ -123,22 +120,23 @@ def run_query(sql, con_args, show_error=True): i = 0 for line in std.splitlines(): if i == 0: - cols = [name for name in line.split(delimiter)] + cols = [name.strip() for name in line.split(delimiter)] else: row = {} # dict of col_name:col_value pairs c = 0 for val in line.split(delimiter): - row[cols[c]] = val + row[cols[c]] = val.strip() c += 1 results.insert(i, row) i += 1 - # Drop the last line: "(X rows)" + # Drop the last 2 lines: "(X rows), ("\n")" try: results.pop() + results.pop() except Exception: pass - return results + return results[1:] # ------------------------------------------------------------------------------ @@ -176,7 +174,11 @@ def get_db_madlib_version(con_args, schema): def get_dbver(con_args, portid): """ Read version number from database (of form X.Y) """ try: - versionStr = run_query("SELECT pg_catalog.version()", con_args, True)[0]['version'] + versionStr = run_query("SELECT pg_catalog.version()", con_args, True) + versionStr = versionStr[0]['version'] + if portid == 'opengauss' and ('openGauss' in versionStr or 'GaussDB' in versionStr): + return '9.2' + if portid == 'postgres': match = re.search("PostgreSQL[a-zA-Z\s]*(\d+\.\d+)", versionStr) elif portid == 'greenplum': diff --git a/src/ports/CMakeLists.txt b/src/ports/CMakeLists.txt index bb08e83..f7ec404 100644 --- a/src/ports/CMakeLists.txt +++ b/src/ports/CMakeLists.txt @@ -1,2 +1,2 @@ add_subdirectory(postgres) -add_subdirectory(greenplum) +#add_subdirectory(greenplum) diff --git a/src/ports/postgres/CMakeLists.txt b/src/ports/postgres/CMakeLists.txt index 6c7b3a8..6919024 100644 --- a/src/ports/postgres/CMakeLists.txt +++ b/src/ports/postgres/CMakeLists.txt @@ -59,7 +59,7 @@ list(APPEND MAD_DBAL_SOURCES # BEGIN Legacy Code file(GLOB_RECURSE LEGACY_C_FILES - "${CMAKE_SOURCE_DIR}/methods/*.c") + "${CMAKE_SOURCE_DIR}/methods/*.cpp") list(APPEND MAD_DBAL_SOURCES ${LEGACY_C_FILES}) # END Legacy Code @@ -154,7 +154,7 @@ function(add_${PORT_LC}_library IN_PORT_VERSION) # The warning GCC would output without -fno-strict-aliasing is: # dereferencing type-punned pointer will break strict-aliasing rules set_source_files_properties(${LEGACY_C_FILES} PROPERTIES - COMPILE_FLAGS "-fno-strict-aliasing") + COMPILE_FLAGS "-fno-strict-aliasing -fsigned-char") endif(CMAKE_COMPILER_IS_GNUCC) # errOmitLocation is used in legacy C code, but it is Greenplum only @@ -226,6 +226,11 @@ function(add_${PORT_LC}_library IN_PORT_VERSION) DESTINATION ports/${PORT_DIR_NAME}/${IN_PORT_VERSION}/madpack COMPONENT ${DBMS} ) + # copy libmadlib.so to $GAUSSHOME/lib/postgresql + MESSAGE(STATUS "HOME dir: $ENV{GAUSSHOME}") + if(DEFINED ENV{GAUSSHOME}) + install(TARGETS madlib_${DBMS} LIBRARY DESTINATION $ENV{GAUSSHOME}/lib/postgresql) + endif() endfunction(add_${PORT_LC}_library) diff --git a/src/ports/postgres/dbconnector/AnyType_impl.hpp b/src/ports/postgres/dbconnector/AnyType_impl.hpp index 41fcd47..4f0011c 100644 --- a/src/ports/postgres/dbconnector/AnyType_impl.hpp +++ b/src/ports/postgres/dbconnector/AnyType_impl.hpp @@ -258,7 +258,7 @@ AnyType::numFields() const { case Scalar: return 1; case ReturnComposite: return static_cast(mChildren.size()); case FunctionComposite: return PG_NARGS(); - case NativeComposite: return HeapTupleHeaderGetNatts(mTupleHeader); + case NativeComposite: return ((mTupleHeader)->t_infomask2 & HEAP_NATTS_MASK); //HeapTupleHeaderGetNatts1(mTupleHeader); default: // This should never happen throw std::logic_error("Unhandled case in AnyType::numFields()."); diff --git a/src/ports/postgres/dbconnector/dbconnector.hpp b/src/ports/postgres/dbconnector/dbconnector.hpp index 523fd1c..e6222e0 100644 --- a/src/ports/postgres/dbconnector/dbconnector.hpp +++ b/src/ports/postgres/dbconnector/dbconnector.hpp @@ -18,9 +18,7 @@ // (commit https://github.com/postgres/postgres/commit/a919937f112eb2f548d5f9bd1b3a7298375e6380) // Since we don't need anything from ports.h we can cheat and say its already been declared. // Warning: This could cause problems in the future... -#define PG_PORT_H -extern "C" { #include #include // Use the macro defined in the header to detect the platform #include @@ -40,7 +38,6 @@ extern "C" { #include // for direct access to catalog, e.g., SearchSysCache() #include // type conversion, e.g., lookup_rowtype_tupdesc #include "../../../../methods/svec/src/pg_gp/sparse_vector.h" // Legacy sparse vectors -} // extern "C" #include "Compatibility.hpp" @@ -77,6 +74,10 @@ extern "C" { #undef dngettext #endif +#ifdef PM +#undef PM +#endif + // Note: If errors occur in the following include files, it could indicate that // new macros have been added to PostgreSQL header files. #include diff --git a/src/ports/postgres/dbconnector/main.cpp b/src/ports/postgres/dbconnector/main.cpp index 1422780..160b11d 100644 --- a/src/ports/postgres/dbconnector/main.cpp +++ b/src/ports/postgres/dbconnector/main.cpp @@ -11,9 +11,7 @@ #include #include -extern "C" { - PG_MODULE_MAGIC; -} // extern "C" +PG_MODULE_MAGIC; namespace madlib { diff --git a/src/ports/postgres/modules/assoc_rules/assoc_rules.py_in b/src/ports/postgres/modules/assoc_rules/assoc_rules.py_in index 6b7a7c9..e21268b 100644 --- a/src/ports/postgres/modules/assoc_rules/assoc_rules.py_in +++ b/src/ports/postgres/modules/assoc_rules/assoc_rules.py_in @@ -12,6 +12,7 @@ import plpy from utilities.validate_args import columns_exist_in_table from utilities.validate_args import table_exists from utilities.control import MinWarning +from utilities.utilities import unique_string """ @@ -144,7 +145,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, plpy.execute(""" CREATE TEMP TABLE assoc_rules_aux_tmp ( - ruleId SERIAL, + ruleId BIGINT, pre TEXT, post TEXT, support FLOAT8, @@ -165,17 +166,17 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, plpy.execute(""" CREATE TEMP TABLE assoc_input_unique AS SELECT - dense_rank() OVER (ORDER BY tid)::BIGINT as tid, + dense_rank() OVER (ORDER BY artid)::BIGINT as artid, item FROM ( - SELECT {0} as tid, + SELECT {0} as artid, {1}::text as item FROM {2} WHERE {3} IS NOT NULL AND {4} IS NOT NULL GROUP BY 1,2 ) t - m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (tid)') + m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (artid)') """.format(tid_col, item_col, input_table, tid_col, item_col) ); @@ -183,7 +184,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, plpy.info("finished removing duplicates"); rv = plpy.execute(""" - SELECT count(DISTINCT tid) as c1, count(DISTINCT item) as c2 + SELECT count(DISTINCT artid) as c1, count(DISTINCT item) as c2 FROM assoc_input_unique """); num_tranx = rv[0]["c1"]; @@ -203,7 +204,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, item::TEXT, cnt::FLOAT8 FROM ( - SELECT item AS item, count(tid) as cnt + SELECT item AS item, count(artid) as cnt FROM assoc_input_unique GROUP BY item ) t @@ -219,11 +220,11 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, plpy.execute("DROP TABLE IF EXISTS assoc_enc_input"); plpy.execute(""" - CREATE TEMP TABLE assoc_enc_input (tid, item, cnt) AS - SELECT t1.tid, t2.item_id, t2.cnt + CREATE TEMP TABLE assoc_enc_input (artid, item, cnt) AS + SELECT t1.artid, t2.item_id, t2.cnt FROM assoc_input_unique t1, assoc_item_uniq t2 WHERE t1.item = t2.item_text - m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (tid)') + m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (artid)') """); if verbose : @@ -281,7 +282,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, (ARRAY[t.item], ARRAY[1], {1}, 0), t.cnt::FLOAT8 / {2}, {3}.svec_cast_positions_float8arr - (array_agg(t.tid), array_agg(1), {4}, 0) + (array_agg(t.artid), array_agg(1), {4}, 0) FROM assoc_enc_input t GROUP BY t.item, t.cnt """.format(madlib_schema, num_supp_prod, num_tranx, madlib_schema, @@ -314,7 +315,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, plpy.execute(""" CREATE TEMP TABLE assoc_loop_aux ( - id BIGSERIAL, + id BIGINT, set_list {0}.svec, support FLOAT8, tids {1}.svec @@ -380,13 +381,15 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, # The iterp1 check ensures that the new itemset is of a certain size. # At every iteration, we increase the target itemset size by one. - plpy.execute("ALTER SEQUENCE assoc_loop_aux_id_seq RESTART WITH 1"); + # plpy.execute("ALTER SEQUENCE assoc_loop_aux_id_seq RESTART WITH 1"); + assoc_loop_aux_sequence = unique_string(desp='s1') plpy.execute("TRUNCATE TABLE assoc_loop_aux"); + plpy.execute("""CREATE SEQUENCE {s} START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; + """.format(s=assoc_loop_aux_sequence)) plpy.execute(""" - INSERT INTO assoc_loop_aux(set_list, support, tids) - SELECT DISTINCT ON({madlib_schema}.svec_to_string(set_list)) set_list, - {madlib_schema}.svec_l1norm(tids)::FLOAT8 / {num_tranx}, - tids + INSERT INTO assoc_loop_aux(set_list, support, tids, id) + SELECT set_list, {madlib_schema}.svec_l1norm(tids)::FLOAT8 / {num_tranx}, tids, nextval('{assoc_loop_aux_sequence}') FROM( + SELECT DISTINCT ON({madlib_schema}.svec_to_string(set_list)) set_list, tids FROM ( SELECT {madlib_schema}.svec_minus( @@ -401,9 +404,11 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, ) t WHERE {madlib_schema}.svec_l1norm(set_list)::INT = {iterp1} AND {madlib_schema}.svec_l1norm(tids)::FLOAT8 >= {min_supp_tranx} + ) q """.format(iterp1=iter + 1, **locals()) ); + plpy.execute("""DROP SEQUENCE IF EXISTS {s};""".format(s=assoc_loop_aux_sequence)) plpy.execute("TRUNCATE TABLE assoc_rule_sets_loop"); plpy.execute(""" INSERT INTO assoc_rule_sets_loop(id, set_list, support, tids) @@ -470,10 +475,14 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, begin_step_exec = time.time(); # generate all the final rules + assoc_rules_aux_tmp_sequence = unique_string(desp='s2') + plpy.execute("""CREATE SEQUENCE {s} START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; + """.format(s=assoc_rules_aux_tmp_sequence)) plpy.execute(""" INSERT INTO assoc_rules_aux_tmp - (pre, post, support, confidence, lift, conviction) + (ruleId, pre, post, support, confidence, lift, conviction) SELECT + nextval('{assoc_rules_aux_tmp_sequence}'), t.item[1], t.item[2], t.support_xy, @@ -499,6 +508,7 @@ def assoc_rules(madlib_schema, support, confidence, tid_col, (t.support_xy / x.support) >= {confidence} """.format(**locals()) ); + plpy.execute("""DROP SEQUENCE IF EXISTS {s};""".format(s=assoc_rules_aux_tmp_sequence)) # generate the readable rules plpy.execute("DROP TABLE IF EXISTS pre_tmp_table"); diff --git a/src/ports/postgres/modules/assoc_rules/assoc_rules.sql_in b/src/ports/postgres/modules/assoc_rules/assoc_rules.sql_in index a7f639b..4d8531a 100644 --- a/src/ports/postgres/modules/assoc_rules/assoc_rules.sql_in +++ b/src/ports/postgres/modules/assoc_rules/assoc_rules.sql_in @@ -529,7 +529,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gen_rules_from_cfp INT ) RETURNS SETOF TEXT[] AS 'MODULE_PATHNAME' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/bayes/bayes.py_in b/src/ports/postgres/modules/bayes/bayes.py_in index b97929f..7507515 100644 --- a/src/ports/postgres/modules/bayes/bayes.py_in +++ b/src/ports/postgres/modules/bayes/bayes.py_in @@ -709,7 +709,6 @@ def create_prepared_data(**kwargs): {attr_counts_sql} m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (attr)'); ALTER TABLE {attrCountsSource} ADD PRIMARY KEY (attr); - ANALYZE {attrCountsSource}; DROP TABLE IF EXISTS {attrValuesSource}; CREATE TEMPORARY TABLE {attrValuesSource} @@ -718,7 +717,6 @@ def create_prepared_data(**kwargs): m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (attr, value)'); ALTER TABLE {attrValuesSource} ADD PRIMARY KEY (attr, value); - ANALYZE {attrValuesSource}; """.format( attrCountsSource = kwargs['attrCountsSource'], attrValuesSource = kwargs['attrValuesSource'], @@ -741,7 +739,6 @@ def create_prepared_data(**kwargs): if kwargs['whatToCreate'] == 'TABLE': plpy.execute(""" ALTER TABLE {classPriorsDestName} ADD PRIMARY KEY (class); - ANALYZE {classPriorsDestName}; """.format(**kwargs)) kwargs.update(dict( @@ -786,7 +783,6 @@ def create_prepared_data(**kwargs): if kwargs['whatToCreate'] == 'TABLE': plpy.execute(""" ALTER TABLE {featureProbsDestName} ADD PRIMARY KEY (class, attr, value); - ANALYZE {featureProbsDestName}; DROP TABLE {attrCountsSource}; DROP TABLE {attrValuesSource}; """.format(**kwargs)) @@ -795,7 +791,6 @@ def create_prepared_data(**kwargs): if kwargs['whatToCreate'] == 'TABLE': plpy.execute(""" ALTER TABLE {numericFeatureStatsDestName} ADD PRIMARY KEY (class, attr); - ANALYZE {numericFeatureStatsDestName}; """.format(**kwargs)) @@ -1173,7 +1168,7 @@ def __verify_attr_num(sourceTable, attrColumn, numAttr): Verify the number of attributes for each record is as expected. """ result = plpy.execute(""" - SELECT count(*) size FROM {0} + SELECT count(*) AS size FROM {0} WHERE array_upper({1}, 1) <> {2} """.format(sourceTable, attrColumn, numAttr) ) diff --git a/src/ports/postgres/modules/convex/lmf.sql_in b/src/ports/postgres/modules/convex/lmf.sql_in index e608d0a..700e34f 100644 --- a/src/ports/postgres/modules/convex/lmf.sql_in +++ b/src/ports/postgres/modules/convex/lmf.sql_in @@ -132,7 +132,7 @@ INSERT INTO lmf_data VALUES (999, 10000, 2.0); -# Call the lmf_igd_run() stored procedure.
 DROP TABLE IF EXISTS lmf_model;
-SELECT madlib.lmf_igd_run( 'lmf_model',
+SELECT MADLIB_SCHEMA.lmf_igd_run( 'lmf_model',
                            'lmf_data',
                            'row',
                            'col',
@@ -245,7 +245,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lmf_igd_transition(
         scale_factor    DOUBLE PRECISION)
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE
+LANGUAGE C IMMUTABLE NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lmf_igd_merge(
@@ -253,14 +253,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lmf_igd_merge(
         state2 DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lmf_igd_final(
         state DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 /**
@@ -293,14 +293,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_lmf_igd_distance(
     /*+ state2 */ DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION AS
 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_lmf_igd_result(
     /*+ state */ DOUBLE PRECISION[])
 RETURNS MADLIB_SCHEMA.lmf_result AS
 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT
+LANGUAGE c IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_lmf_igd_args(
@@ -309,7 +309,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_lmf_igd_args(
 ) RETURNS VOID
 IMMUTABLE
 CALLED ON NULL INPUT
-LANGUAGE c
+LANGUAGE c NOT FENCED
 AS 'MODULE_PATHNAME', 'exec_sql_using'
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
@@ -382,8 +382,9 @@ BEGIN
     -- PostgreSQL 8.4:
     -- http://www.postgresql.org/docs/8.4/static/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN
     -- We therefore have to emulate.
+    DROP TABLE IF EXISTS pg_temp._madlib_lmf_igd_args;
+
     PERFORM MADLIB_SCHEMA.internal_execute_using_lmf_igd_args($sql$
-        DROP TABLE IF EXISTS pg_temp._madlib_lmf_igd_args;
         CREATE TABLE pg_temp._madlib_lmf_igd_args AS
         SELECT
             $1 AS row_dim,
diff --git a/src/ports/postgres/modules/convex/mlp.sql_in b/src/ports/postgres/modules/convex/mlp.sql_in
index d6ce7ce..6caaae5 100644
--- a/src/ports/postgres/modules/convex/mlp.sql_in
+++ b/src/ports/postgres/modules/convex/mlp.sql_in
@@ -1442,7 +1442,7 @@ CREATE FUNCTION MADLIB_SCHEMA.mlp_igd_transition(
     )
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE;
+LANGUAGE C IMMUTABLE NOT FENCED;
 
 CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_transition(
         state              DOUBLE PRECISION[],
@@ -1463,33 +1463,33 @@ CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_transition(
     )
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE;
+LANGUAGE C IMMUTABLE NOT FENCED;
 
 CREATE FUNCTION MADLIB_SCHEMA.mlp_igd_merge(
         state1 DOUBLE PRECISION[],
         state2 DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
 
 CREATE FUNCTION MADLIB_SCHEMA.mlp_igd_final(
         state DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
 
 CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_merge(
         state1 DOUBLE PRECISION[],
         state2 DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
 
 CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_final(
         state DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
 
 /**
  * @internal
@@ -1550,19 +1550,19 @@ CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_igd_distance(
     /*+ state2 */ DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION AS
 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
 
 CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_igd_result(
     /*+ state */ DOUBLE PRECISION[])
 RETURNS MADLIB_SCHEMA.mlp_result AS
 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
 
 CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_minibatch_result(
     /*+ state */ DOUBLE PRECISION[])
 RETURNS MADLIB_SCHEMA.mlp_result AS
 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
 -------------------------------------------------------------------------
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlp_classification(
@@ -1834,7 +1834,7 @@ CREATE FUNCTION MADLIB_SCHEMA.internal_predict_mlp(
     )
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
 
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlp_classification(
diff --git a/src/ports/postgres/modules/convex/utils_regularization.sql_in b/src/ports/postgres/modules/convex/utils_regularization.sql_in
index 0bcef63..a4dc35f 100644
--- a/src/ports/postgres/modules/convex/utils_regularization.sql_in
+++ b/src/ports/postgres/modules/convex/utils_regularization.sql_in
@@ -12,7 +12,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_result (
     in_state            DOUBLE PRECISION[]
 ) RETURNS MADLIB_SCHEMA.__utils_scales AS
 'MODULE_PATHNAME', '__utils_var_scales_result'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 ------------------------------------------------------------------------
@@ -23,7 +23,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_transition (
     dimension           INTEGER
 ) RETURNS DOUBLE PRECISION[] AS
 'MODULE_PATHNAME', 'utils_var_scales_transition'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 --
@@ -33,7 +33,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_merge (
     state2              DOUBLE PRECISION[]
 ) RETURNS DOUBLE PRECISION[] AS
 'MODULE_PATHNAME', 'utils_var_scales_merge'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 --
@@ -42,14 +42,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_final (
     state               DOUBLE PRECISION[]
 ) RETURNS DOUBLE PRECISION[] AS
 'MODULE_PATHNAME', 'utils_var_scales_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__utils_var_scales_non_zero_std_final (
     state               DOUBLE PRECISION[]
 ) RETURNS DOUBLE PRECISION[] AS
 'MODULE_PATHNAME', 'utils_var_scales_non_zero_std_final'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
 
 --
@@ -88,5 +88,5 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.utils_normalize_data (
     std         DOUBLE PRECISION[]
 ) RETURNS DOUBLE PRECISION[] AS
 'MODULE_PATHNAME', 'utils_normalize_data'
-LANGUAGE C IMMUTABLE STRICT
+LANGUAGE C IMMUTABLE STRICT NOT FENCED
 m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
diff --git a/src/ports/postgres/modules/crf/crf.py_in b/src/ports/postgres/modules/crf/crf.py_in
index 2eaa1e8..e812134 100644
--- a/src/ports/postgres/modules/crf/crf.py_in
+++ b/src/ports/postgres/modules/crf/crf.py_in
@@ -209,7 +209,7 @@ def lincrf_train(schema_madlib, train_feature_tbl, train_featureset_tbl,
                             schema_madlib = schema_madlib))
 
     # Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
-    plpy.execute(""" ANALYZE {crf_weights_tbl} """.format(crf_weights_tbl = crf_weights_tbl))
+    # plpy.execute(""" ANALYZE {crf_weights_tbl} """.format(crf_weights_tbl = crf_weights_tbl))
 
     return "CRF Train successful. Results stored in the specified CRF stats and weights table"
 
diff --git a/src/ports/postgres/modules/crf/crf.sql_in b/src/ports/postgres/modules/crf/crf.sql_in
index f9f6358..230df8e 100644
--- a/src/ports/postgres/modules/crf/crf.sql_in
+++ b/src/ports/postgres/modules/crf/crf.sql_in
@@ -708,33 +708,33 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lincrf_lbfgs_step_transition(
     DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE;
+LANGUAGE C IMMUTABLE NOT FENCED;
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lincrf_lbfgs_step_merge_states(
     state1 DOUBLE PRECISION[],
     state2 DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lincrf_lbfgs_step_final(
     state DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
-LANGUAGE C IMMUTABLE STRICT;
+LANGUAGE C IMMUTABLE STRICT NOT FENCED;
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_lincrf_lbfgs_converge(
     /*+ state */ DOUBLE PRECISION[])
 RETURNS DOUBLE PRECISION AS
 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
 
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_lincrf_lbfgs_result(
     /*+ state */ DOUBLE PRECISION[])
 RETURNS MADLIB_SCHEMA.lincrf_result AS
 'MODULE_PATHNAME'
-LANGUAGE c IMMUTABLE STRICT;
+LANGUAGE c IMMUTABLE STRICT NOT FENCED;
 
 /**
  * @internal
diff --git a/src/ports/postgres/modules/crf/crf_feature_gen.py_in b/src/ports/postgres/modules/crf/crf_feature_gen.py_in
index f55037f..7f9857b 100644
--- a/src/ports/postgres/modules/crf/crf_feature_gen.py_in
+++ b/src/ports/postgres/modules/crf/crf_feature_gen.py_in
@@ -138,7 +138,7 @@ def generate_train_features(schema_madlib, train_segment_tbl, regex_tbl, label_t
                     FROM   """ + tmp1_feature + """;""")
 
     # Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
-    plpy.execute("""ANALYZE {tmp1_feature} """.format(tmp1_feature = tmp1_feature))
+    # plpy.execute("""ANALYZE {tmp1_feature} """.format(tmp1_feature = tmp1_feature))
 
     plpy.execute("""DROP SEQUENCE IF EXISTS seq;
                     CREATE  SEQUENCE seq START 1 INCREMENT 1;""")
@@ -152,7 +152,7 @@ def generate_train_features(schema_madlib, train_segment_tbl, regex_tbl, label_t
                    tmp_featureset = tmp_featureset))
 
     # Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
-    plpy.execute(""" ANALYZE {train_featureset_tbl} """.format(train_featureset_tbl = train_featureset_tbl))
+    # plpy.execute(""" ANALYZE {train_featureset_tbl} """.format(train_featureset_tbl = train_featureset_tbl))
 
     rv = plpy.execute("""SELECT COUNT(*) AS total_feature FROM """ + train_featureset_tbl + """;""")
 
@@ -205,7 +205,7 @@ def generate_train_features(schema_madlib, train_segment_tbl, regex_tbl, label_t
                 dense_mtbl = dense_mtbl))
 
     # Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
-    plpy.execute(""" ANALYZE {train_feature_tbl} """.format(train_feature_tbl = train_feature_tbl))
+    # plpy.execute(""" ANALYZE {train_feature_tbl} """.format(train_feature_tbl = train_feature_tbl))
 
     plpy.execute("""SET client_min_messages TO """ + str(origClientMinMessages[0]['setting']) + """;""")
 
@@ -412,8 +412,8 @@ m4_ifdef(`__HAS_ORDERED_AGGREGATES__', `
                         GROUP BY seg_text,label;""")
 
     # Enforce ANALYZE to gather proper table statistics required to generate optimized query plans
-    plpy.execute(""" ANALYZE {viterbi_mtbl} """.format(viterbi_mtbl = viterbi_mtbl))
-    plpy.execute(""" ANALYZE {viterbi_rtbl} """.format(viterbi_rtbl = viterbi_rtbl))
+    # plpy.execute(""" ANALYZE {viterbi_mtbl} """.format(viterbi_mtbl = viterbi_mtbl))
+    # plpy.execute(""" ANALYZE {viterbi_rtbl} """.format(viterbi_rtbl = viterbi_rtbl))
 
 
 def _validate_label_tbl(label_tbl):
diff --git a/src/ports/postgres/modules/crf/viterbi.sql_in b/src/ports/postgres/modules/crf/viterbi.sql_in
index d3c20cd..8aa0b08 100644
--- a/src/ports/postgres/modules/crf/viterbi.sql_in
+++ b/src/ports/postgres/modules/crf/viterbi.sql_in
@@ -35,7 +35,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
  * @returns the top1 label sequence, the last two elements in the array is used to calculate the top1 probability
  */
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.vcrf_top1_label(mArray DOUBLE PRECISION[], rArray DOUBLE PRECISION[], nlabel int)
-returns int[] as 'MODULE_PATHNAME' language c strict;
+returns int[] as 'MODULE_PATHNAME' language c strict NOT FENCED;
 
 
 /**
diff --git a/src/ports/postgres/modules/deep_learning/gpu_info_from_tf.py_in b/src/ports/postgres/modules/deep_learning/gpu_info_from_tf.py_in
deleted file mode 100644
index b1e59b1..0000000
--- a/src/ports/postgres/modules/deep_learning/gpu_info_from_tf.py_in
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-@file gpu_info_from_tf.py_in
-
-@brief This file prints out the gpu descriptions using tensorflow API. This file
-is intended to be called using subprocess. See madlib_keras_gpu_info.py_in
-for more details.
-"""
-
-from tensorflow.python.client import device_lib
-from keras import backend as K
-
-config = K.tf.ConfigProto()
-config.gpu_options.allow_growth = True
-sess = K.tf.Session(config=config)
-local_device_protos = device_lib.list_local_devices()
-K.clear_session()
-sess.close()
-if local_device_protos:
-    for x in local_device_protos:
-        if x.device_type == 'GPU':
-            print x.physical_device_desc
diff --git a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
deleted file mode 100644
index 605d439..0000000
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
+++ /dev/null
@@ -1,1036 +0,0 @@
-# coding=utf-8
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-"""
-@file input_data_preprocessor.py_in
-
-"""
-from math import ceil
-import plpy
-
-from internal.db_utils import get_distinct_col_levels
-from internal.db_utils import quote_literal
-from internal.db_utils import get_product_of_dimensions
-from utilities.minibatch_preprocessing import MiniBatchBufferSizeCalculator
-from utilities.control import OptimizerControl
-from utilities.control import HashaggControl
-from utilities.utilities import _assert
-from utilities.utilities import add_postfix
-from utilities.utilities import is_platform_pg
-from utilities.utilities import is_psql_char_type
-from utilities.utilities import is_valid_psql_type
-from utilities.utilities import is_var_valid
-from utilities.utilities import BOOLEAN, NUMERIC, ONLY_ARRAY, TEXT
-from utilities.utilities import py_list_to_sql_string
-from utilities.utilities import split_quoted_delimited_str
-from utilities.utilities import strip_end_quotes
-from utilities.utilities import unique_string
-from utilities.utilities import validate_module_input_params
-from utilities.utilities import get_seg_number
-from utilities.validate_args import input_tbl_valid
-from utilities.validate_args import get_expr_type
-
-from madlib_keras_helper import *
-import time
-
-NUM_CLASSES_COLNAME = "num_classes"
-
-class InputDataPreprocessorDL(object):
-    def __init__(self, schema_madlib, source_table, output_table,
-                 dependent_varname, independent_varname, buffer_size,
-                 normalizing_const, num_classes, distribution_rules, module_name):
-        self.schema_madlib = schema_madlib
-        self.source_table = source_table
-        self.output_table = output_table
-        self.dependent_varname = dependent_varname
-        self.independent_varname = independent_varname
-        self.buffer_size = buffer_size
-        self.normalizing_const = normalizing_const
-        self.num_classes = num_classes
-        self.distribution_rules = distribution_rules if distribution_rules else 'all_segments'
-        self.module_name = module_name
-        self.output_summary_table = None
-        self.dependent_vartype = None
-        self.independent_vartype = None
-        self.gpu_config = '$__madlib__$all_segments$__madlib__$'
-        if self.output_table:
-            self.output_summary_table = add_postfix(self.output_table, "_summary")
-
-        ## Validating input args prior to using them in _set_validate_vartypes()
-        self._validate_args()
-        self._set_validate_vartypes()
-        self.dependent_levels = None
-        # The number of padded zeros to include in 1-hot vector
-        self.padding_size = 0
-
-    def _set_one_hot_encoding_variables(self):
-        """
-            Set variables such as dependent_levels and padding_size.
-            If necessary, NULLs are padded to dependent_levels list.
-        """
-        if self.dependent_levels:
-            # if any class level was NULL in sql, that would show up as
-            # None in self.dependent_levels. Replace all None with NULL
-            # in the list.
-            self.dependent_levels = ['NULL' if level is None else level
-                for level in self.dependent_levels]
-            self._validate_num_classes()
-            # Try computing padding_size after running all necessary validations.
-            if self.num_classes:
-                self.padding_size = self.num_classes - len(self.dependent_levels)
-
-    def _validate_num_classes(self):
-        if self.num_classes is not None and \
-            self.num_classes < len(self.dependent_levels):
-            plpy.error("{0}: Invalid num_classes value specified. It must "\
-                "be equal to or greater than distinct class values found "\
-                "in table ({1}).".format(
-                    self.module_name, len(self.dependent_levels)))
-
-    def _validate_distribution_table(self):
-
-        input_tbl_valid(self.distribution_rules, self.module_name,
-                        error_suffix_str="""
-                        segments_to_use table ({self.distribution_rules}) doesn't exist.
-                        """.format(self=self))
-        _assert(is_var_valid(self.distribution_rules, 'dbid'),
-                "{self.module_name}: distribution rules table must contain dbib column".format(
-                    self=self))
-        dbids = plpy.execute("""
-            SELECT array_agg(dbid) AS dbids FROM gp_segment_configuration
-            WHERE content >= 0 AND role = 'p'
-            """)[0]['dbids']
-        dist_result = plpy.execute("""
-            SELECT array_agg(dbid) AS dbids,
-                   count(dbid) AS c1,
-                   count(DISTINCT dbid) AS c2
-            FROM {0} """.format(self.distribution_rules))
-
-        _assert(dist_result[0]['c1'] == dist_result[0]['c2'],
-            '{self.module_name}: distribution rules table contains duplicate dbids'.format(
-                self=self))
-
-        for i in dist_result[0]['dbids']:
-            _assert(i in dbids,
-                '{self.module_name}: invalid dbid:{i} in the distribution rules table'.format(
-                    self=self, i=i))
-
-    def get_one_hot_encoded_dep_var_expr(self):
-        """
-        :param dependent_varname: Name of the dependent variable
-        :param num_classes: Number of class values to consider in 1-hot
-        :return:
-            This function returns a tuple of
-            1. A string with transformed dependent varname depending on it's type
-            2. All the distinct dependent class levels encoded as a string
-
-            If dep_type == numeric[] , do not encode
-                    1. dependent_varname = rings
-                        transformed_value = ARRAY[rings]
-                    2. dependent_varname = ARRAY[a, b, c]
-                        transformed_value = ARRAY[a, b, c]
-            else if dep_type in ("text", "boolean"), encode:
-                    3. dependent_varname = rings (encoding)
-                        transformed_value = ARRAY[rings=1, rings=2, rings=3]
-        """
-        # Assuming the input NUMERIC[] is already one_hot_encoded,
-        # so casting to INTEGER[]
-        if is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY):
-            return "{0}::{1}[]".format(self.dependent_varname, SMALLINT_SQL_TYPE)
-
-        # For DL use case, we want to allow NULL as a valid class value,
-        # so the query must have 'IS NOT DISTINCT FROM' instead of '='
-        # like in the generic get_one_hot_encoded_expr() defined in
-        # db_utils.py_in. We also have this optional 'num_classes' param
-        # that affects the logic of 1-hot encoding. Since this is very
-        # specific to input_preprocessor_dl for now, let's keep
-        # it here instead of refactoring it out to a generic helper function.
-        one_hot_encoded_expr = ["({0}) IS NOT DISTINCT FROM {1}".format(
-            self.dependent_varname, c) for c in self.dependent_levels]
-        if self.num_classes:
-            one_hot_encoded_expr.extend(['false'
-                for i in range(self.padding_size)])
-        # In psql, we can't directly convert boolean to smallint, so we firstly
-        # convert it to integer and then cast to smallint
-        return 'ARRAY[{0}]::INTEGER[]::{1}[]'.format(
-            ', '.join(one_hot_encoded_expr), SMALLINT_SQL_TYPE)
-
-    def _get_independent_var_shape(self):
-
-        shape = plpy.execute(
-            "SELECT array_dims({0}) AS shape FROM {1} LIMIT 1".format(
-            self.independent_varname, self.source_table))[0]['shape']
-        return parse_shape(shape)
-
-    def _get_dependent_var_shape(self):
-
-        if self.num_classes:
-            shape = [self.num_classes]
-        elif self.dependent_levels:
-            shape = [len(self.dependent_levels)]
-        else:
-            shape = plpy.execute(
-                "SELECT array_dims({0}) AS shape FROM {1} LIMIT 1".format(
-                self.dependent_varname, self.source_table))[0]['shape']
-            shape = parse_shape(shape)
-        return shape
-
-    def input_preprocessor_dl(self, order_by_random=True):
-        """
-            Creates the output and summary table that does the following
-            pre-processing operations on the input data:
-            1) Normalizes the independent variable.
-            2) Minibatches the normalized independent variable.
-            3) One-hot encodes the dependent variable.
-            4) Minibatches the one-hot encoded dependent variable.
-        """
-        # setup for 1-hot encoding
-        self._set_one_hot_encoding_variables()
-
-        # Generate random strings for TEMP tables
-        series_tbl = unique_string(desp='series')
-        dist_key_tbl = unique_string(desp='dist_key')
-        normalized_tbl = unique_string(desp='normalized_table')
-        batched_table = unique_string(desp='batched_table')
-
-        # Used later in locals() for formatting queries
-        x=MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL
-        y=MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL
-        float32=FLOAT32_SQL_TYPE
-        dep_shape_col = add_postfix(y, "_shape")
-        ind_shape_col = add_postfix(x, "_shape")
-
-        ind_shape = self._get_independent_var_shape()
-        ind_shape = ','.join([str(i) for i in ind_shape])
-        dep_shape = self._get_dependent_var_shape()
-        dep_shape = ','.join([str(i) for i in dep_shape])
-
-        one_hot_dep_var_array_expr = self.get_one_hot_encoded_dep_var_expr()
-
-        # skip normalization step if normalizing_const = 1.0
-        if self.normalizing_const and (self.normalizing_const < 0.999999 or self.normalizing_const > 1.000001):
-            rescale_independent_var = """{self.schema_madlib}.array_scalar_mult(
-                                         {self.independent_varname}::{float32}[],
-                                         (1/{self.normalizing_const})::{float32})
-                                      """.format(**locals())
-        else:
-            self.normalizing_const = DEFAULT_NORMALIZING_CONST
-            rescale_independent_var = "{self.independent_varname}::{float32}[]".format(**locals())
-
-        # It's important that we shuffle all rows before batching for fit(), but
-        #  we can skip that for predict()
-        order_by_clause = " ORDER BY RANDOM()" if order_by_random else ""
-
-        # This query template will be used later in pg & gp specific code paths,
-        #  where {make_buffer_id} and {dist_by_buffer_id} are filled in
-        batching_query = """
-            CREATE TEMP TABLE {batched_table} AS SELECT
-                {{make_buffer_id}} buffer_id,
-                {self.schema_madlib}.agg_array_concat(
-                    ARRAY[x_norm::{float32}[]]) AS {x},
-                {self.schema_madlib}.agg_array_concat(
-                    ARRAY[y]) AS {y},
-                COUNT(*) AS count
-            FROM {normalized_tbl}
-            GROUP BY buffer_id
-            {{dist_by_buffer_id}}
-        """.format(**locals())
-
-        # This query template will be used later in pg & gp specific code paths,
-        #  where {dist_key_col_comma} and {dist_by_dist_key} will be filled in
-        bytea_query = """
-            CREATE TABLE {self.output_table} AS SELECT
-                {{dist_key_col_comma}}
-                {self.schema_madlib}.array_to_bytea({x}) AS {x},
-                {self.schema_madlib}.array_to_bytea({y}) AS {y},
-                ARRAY[count,{ind_shape}]::SMALLINT[] AS {ind_shape_col},
-                ARRAY[count,{dep_shape}]::SMALLINT[] AS {dep_shape_col},
-                buffer_id
-            FROM {batched_table}
-            {{dist_by_dist_key}}
-        """.format(**locals())
-
-        if is_platform_pg():
-            # used later for writing summary table
-            self.distribution_rules = '$__madlib__$all_segments$__madlib__$'
-
-            #
-            # For postgres, we just need 3 simple queries:
-            #   1-hot-encode/normalize + batching + bytea conversion
-            #
-
-            # see note in gpdb code branch (lower down) on
-            # 1-hot-encoding of dependent var
-            one_hot_sql = """
-                CREATE TEMP TABLE {normalized_tbl} AS SELECT
-                    (ROW_NUMBER() OVER({order_by_clause}) - 1)::INTEGER as row_id,
-                    {rescale_independent_var} AS x_norm,
-                    {one_hot_dep_var_array_expr} AS y
-                FROM {self.source_table}
-            """.format(**locals())
-
-            plpy.execute(one_hot_sql)
-
-            self.buffer_size = self._get_buffer_size(1)
-
-            # Used to format query templates with locals()
-            make_buffer_id = 'row_id / {0} AS '.format(self.buffer_size)
-            dist_by_dist_key = ''
-            dist_by_buffer_id = ''
-            dist_key_col_comma = ''
-
-            # Disable hashagg since large number of arrays being concatenated
-            # could result in excessive memory usage.
-            with HashaggControl(False):
-                # Batch rows with GROUP BY
-                plpy.execute(batching_query.format(**locals()))
-
-            plpy.execute("DROP TABLE {0}".format(normalized_tbl))
-
-            # Convert to BYTEA and output final (permanent table)
-            plpy.execute(bytea_query.format(**locals()))
-
-            plpy.execute("DROP TABLE {0}".format(batched_table))
-
-            self._create_output_summary_table()
-
-            return
-
-        # Done with postgres, rest is all for gpdb
-        #
-        # This gpdb code path is far more complex, and depends on
-        #   how the user wishes to distribute the data.  Even if
-        #   it's to be spread evenly across all segments, we still
-        #   need to do some extra work to ensure that happens.
-
-        if self.distribution_rules == 'all_segments':
-            all_segments = True
-            self.distribution_rules = '$__madlib__$all_segments$__madlib__$'
-            num_segments = get_seg_number()
-        else:
-            all_segments = False
-
-        if self.distribution_rules == 'gpu_segments':
-            #TODO can we reuse the function `get_accessible_gpus_for_seg` from
-            # madlib_keras_helper
-            gpu_info_table = unique_string(desp='gpu_info')
-            plpy.execute("""
-                SELECT {self.schema_madlib}.gpu_configuration('{gpu_info_table}')
-            """.format(**locals()))
-            gpu_query = """
-                SELECT array_agg(DISTINCT(hostname)) as gpu_config
-                FROM {gpu_info_table}
-            """.format(**locals())
-            gpu_query_result = plpy.execute(gpu_query)[0]['gpu_config']
-            if not gpu_query_result:
-               plpy.error("{self.module_name}: No GPUs configured on hosts.".format(self=self))
-            plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
-
-            gpu_config_hostnames = "ARRAY{0}".format(gpu_query_result)
-            # find hosts with gpus
-            get_segment_query = """
-                SELECT array_agg(content) as segment_ids,
-                       array_agg(dbid) as dbid,
-                       count(*) as count
-                FROM gp_segment_configuration
-                WHERE content != -1 AND role = 'p'
-                AND hostname=ANY({gpu_config_hostnames})
-            """.format(**locals())
-            segment_ids_result = plpy.execute(get_segment_query)[0]
-
-            self.gpu_config = "ARRAY{0}".format(sorted(segment_ids_result['segment_ids']))
-            self.distribution_rules = "ARRAY{0}".format(sorted(segment_ids_result['dbid']))
-
-            num_segments = segment_ids_result['count']
-
-        elif not all_segments:  # Read from a table with dbids to distribute the data
-            self._validate_distribution_table()
-            gpu_query = """
-                SELECT array_agg(content) as gpu_config,
-                       array_agg(gp_segment_configuration.dbid) as dbid
-                FROM {self.distribution_rules} JOIN gp_segment_configuration
-                ON {self.distribution_rules}.dbid = gp_segment_configuration.dbid
-            """.format(**locals())
-            gpu_query_result = plpy.execute(gpu_query)[0]
-            self.gpu_config = "ARRAY{0}".format(sorted(gpu_query_result['gpu_config']))
-            num_segments = plpy.execute("SELECT count(*) as count FROM {self.distribution_rules}".format(**locals()))[0]['count']
-            self.distribution_rules = "ARRAY{0}".format(sorted(gpu_query_result['dbid']))
-
-        join_key = 't.buffer_id % {num_segments}'.format(**locals())
-
-        if not all_segments:
-            join_key = '({self.gpu_config})[{join_key} + 1]'.format(**locals())
-
-        # Create large temp table such that there is atleast 1 row on each segment
-        # Using 999999 would distribute data(atleast 1 row on each segment) for
-        # a cluster as large as 20000
-        dist_key_col = DISTRIBUTION_KEY_COLNAME
-        query = """
-            CREATE TEMP TABLE {series_tbl} AS
-                SELECT generate_series(0, 999999) {dist_key_col}
-                DISTRIBUTED BY ({dist_key_col})
-            """.format(**locals())
-
-        plpy.execute(query)
-
-        # Used in locals() to format queries, including template queries
-        #  bytea_query & batching_query defined in section common to
-        #  pg & gp (very beginning of this function)
-        dist_by_dist_key = 'DISTRIBUTED BY ({dist_key_col})'.format(**locals())
-        dist_by_buffer_id = 'DISTRIBUTED BY (buffer_id)'
-        dist_key_col_comma = dist_key_col + ' ,'
-        make_buffer_id = ''
-
-        dist_key_query = """
-                CREATE TEMP TABLE {dist_key_tbl} AS
-                SELECT min({dist_key_col}) AS {dist_key_col}
-                FROM {series_tbl}
-                GROUP BY gp_segment_id
-                DISTRIBUTED BY ({dist_key_col})
-        """.format(**locals())
-
-        plpy.execute(dist_key_query)
-
-        plpy.execute("DROP TABLE {0}".format(series_tbl))
-
-        # Always one-hot encode the dependent var. For now, we are assuming
-        # that input_preprocessor_dl will be used only for deep
-        # learning and mostly for classification. So make a strong
-        # assumption that it is only for classification, so one-hot
-        # encode the dep var, unless it's already a numeric array in
-        # which case we assume it's already one-hot encoded.
-
-        # While 1-hot-encoding is done, we also normalize the independent
-        # var and randomly shuffle the rows on each segment.  (The dist key
-        # we're adding avoids any rows moving between segments.  This may
-        # make things slightly less random, but helps with speed--probably
-        # a safe tradeoff to make.)
-
-        norm_tbl = unique_string(desp='norm_table')
-
-        one_hot_sql = """
-            CREATE TEMP TABLE {norm_tbl} AS
-            SELECT {dist_key_col},
-                {rescale_independent_var} AS x_norm,
-                {one_hot_dep_var_array_expr} AS y
-            FROM {self.source_table} s JOIN {dist_key_tbl} AS d
-                ON (s.gp_segment_id = d.gp_segment_id)
-            {order_by_clause}
-            DISTRIBUTED BY ({dist_key_col})
-        """.format(**locals())
-        plpy.execute(one_hot_sql)
-
-        rows_per_seg_tbl = unique_string(desp='rows_per_seg')
-        start_rows_tbl = unique_string(desp='start_rows')
-
-        #  Generate rows_per_segment table; this small table will
-        #  just have one row on each segment containing the number
-        #  of rows on that segment in the norm_tbl
-        sql = """
-            CREATE TEMP TABLE {rows_per_seg_tbl} AS SELECT
-                COUNT(*) as rows_per_seg,
-                {dist_key_col}
-            FROM {norm_tbl}
-            GROUP BY {dist_key_col}
-            DISTRIBUTED BY ({dist_key_col})
-        """.format(**locals())
-
-        plpy.execute(sql)
-
-        #  Generate start_rows_tbl from rows_per_segment table.
-        #  This assigns a start_row number for each segment based on
-        #  the sum of all rows in previous segments.  These will be
-        #  added to the row numbers within each segment to get an
-        #  absolute index into the table.  All of this is to accomplish
-        #  the equivalent of ROW_NUMBER() OVER() on the whole table,
-        #  but this way is much faster because we don't have to do an
-        #  N:1 Gather Motion (moving entire table to a single segment
-        #  and scanning through it).
-        #
-        sql = """
-            CREATE TEMP TABLE {start_rows_tbl} AS SELECT
-                {dist_key_col},
-                SUM(rows_per_seg) OVER (ORDER BY gp_segment_id) - rows_per_seg AS start_row
-            FROM {rows_per_seg_tbl}
-            DISTRIBUTED BY ({dist_key_col})
-        """.format(**locals())
-
-        plpy.execute(sql)
-
-        plpy.execute("DROP TABLE {0}".format(rows_per_seg_tbl))
-
-        self.buffer_size = self._get_buffer_size(num_segments)
-
-        # The query below assigns slot_id's to each row within
-        #  a segment, computes a row_id by adding start_row for
-        #  that segment to it, then divides by buffer_size to make
-        #  this into a buffer_id
-        # ie:
-        #  buffer_id = row_id / buffer_size
-        #     row_id = start_row + slot_id
-        #    slot_id = ROW_NUMBER() OVER(PARTITION BY )::INTEGER
-        #
-        #   Instead of partitioning by gp_segment_id itself, we
-        # use __dist_key__ col instead.  This is the same partition,
-        # since there's a 1-to-1 mapping between the columns; but
-        # using __dist_key__ avoids an extra Redistribute Motion.
-        #
-        # Note: even though the ordering of these two columns is
-        #  different, this doesn't matter as each segment is being
-        #  numbered separately (only the start_row is different,
-        #  and those are fixed to the correct segments by the JOIN
-        #  condition.
-
-        sql = """
-        CREATE TEMP TABLE {normalized_tbl} AS SELECT
-            {dist_key_col},
-            x_norm,
-            y,
-            (ROW_NUMBER() OVER( PARTITION BY {dist_key_col} ))::INTEGER as slot_id,
-            ((start_row +
-               (ROW_NUMBER() OVER( PARTITION BY {dist_key_col} ) - 1)
-             )::INTEGER / {self.buffer_size}
-            ) AS buffer_id
-        FROM {norm_tbl} JOIN {start_rows_tbl}
-            USING ({dist_key_col})
-        ORDER BY buffer_id
-        DISTRIBUTED BY (slot_id)
-        """.format(**locals())
-
-        plpy.execute(sql)   # label buffer_id's
-
-        # A note on DISTRIBUTED BY (slot_id) in above query:
-        #
-        #     In the next query, we'll be doing the actual batching.  Due
-        #  to the GROUP BY, gpdb will Redistribute on buffer_id.  We could
-        #  avoid this by using DISTRIBUTED BY (buffer_id) in the above
-        #  (buffer-labelling) query.  But this also causes the GROUP BY
-        #  to use single-stage GroupAgg instead of multistage GroupAgg,
-        #  which for unknown reasons is *much* slower and often runs out
-        #  of VMEM unless it's set very high!
-
-        plpy.execute("DROP TABLE {norm_tbl}, {start_rows_tbl}".format(**locals()))
-
-        # Disable optimizer (ORCA) for platforms that use it
-        # since we want to use a groupagg instead of hashagg
-        with OptimizerControl(False):
-            with HashaggControl(False):
-                # Run actual batching query
-                plpy.execute(batching_query.format(**locals()))
-
-        plpy.execute("DROP TABLE {0}".format(normalized_tbl))
-
-        if not all_segments: # remove any segments we don't plan to use
-            sql = """
-                DELETE FROM {dist_key_tbl}
-                    WHERE NOT gp_segment_id = ANY({self.gpu_config})
-            """.format(**locals())
-
-        plpy.execute("ANALYZE {dist_key_tbl}".format(**locals()))
-        plpy.execute("ANALYZE {batched_table}".format(**locals()))
-
-        # Redistribute from buffer_id to dist_key
-        #
-        #  This has to be separate from the batching query, because
-        #   we found that adding DISTRIBUTED BY (dist_key) to that
-        #   query causes it to run out of VMEM on large datasets such
-        #   as places100.  Possibly this is because the memory available
-        #   for GroupAgg has to be shared with an extra slice if they
-        #   are part of the same query.
-        #
-        #  We also tried adding this to the BYTEA conversion query, but
-        #   that resulted in slower performance than just keeping it
-        #   separate.
-        #
-        sql = """CREATE TEMP TABLE {batched_table}_dist_key AS
-                    SELECT {dist_key_col}, t.*
-                        FROM {batched_table} t
-                            JOIN {dist_key_tbl} d
-                                ON {join_key} = d.gp_segment_id
-                            DISTRIBUTED BY ({dist_key_col})
-              """.format(**locals())
-
-        # match buffer_id's with dist_keys
-        plpy.execute(sql)
-
-        sql = """DROP TABLE {batched_table}, {dist_key_tbl};
-                 ALTER TABLE {batched_table}_dist_key RENAME TO {batched_table}
-              """.format(**locals())
-        plpy.execute(sql)
-
-        # Convert batched table to BYTEA and output as final (permanent) table
-        plpy.execute(bytea_query.format(**locals()))
-
-        plpy.execute("DROP TABLE {0}".format(batched_table))
-
-        # Create summary table
-        self._create_output_summary_table()
-
-    def _create_output_summary_table(self):
-        class_level_str='NULL::{0}[]'.format(self.dependent_vartype)
-        if self.dependent_levels:
-            # Update dependent_levels to include NULL when
-            # num_classes > len(self.dependent_levels)
-            if self.num_classes:
-                self.dependent_levels.extend(['NULL'
-                    for i in range(self.padding_size)])
-            else:
-                self.num_classes = len(self.dependent_levels)
-            class_level_str=py_list_to_sql_string(
-                self.dependent_levels, array_type=self.dependent_vartype,
-                long_format=True)
-        if self.num_classes is None:
-            self.num_classes = 'NULL::INTEGER'
-        query = """
-            CREATE TABLE {self.output_summary_table} AS
-            SELECT
-                $__madlib__${self.source_table}$__madlib__$::TEXT AS source_table,
-                $__madlib__${self.output_table}$__madlib__$::TEXT AS output_table,
-                $__madlib__${self.dependent_varname}$__madlib__$::TEXT AS {dependent_varname_colname},
-                $__madlib__${self.independent_varname}$__madlib__$::TEXT AS {independent_varname_colname},
-                $__madlib__${self.dependent_vartype}$__madlib__$::TEXT AS {dependent_vartype_colname},
-                {class_level_str} AS {class_values_colname},
-                {self.buffer_size} AS buffer_size,
-                {self.normalizing_const}::{FLOAT32_SQL_TYPE} AS {normalizing_const_colname},
-                {self.num_classes} AS {num_classes_colname},
-                {self.distribution_rules} AS distribution_rules,
-                {self.gpu_config} AS {internal_gpu_config}
-            """.format(self=self, class_level_str=class_level_str,
-                       dependent_varname_colname=DEPENDENT_VARNAME_COLNAME,
-                       independent_varname_colname=INDEPENDENT_VARNAME_COLNAME,
-                       dependent_vartype_colname=DEPENDENT_VARTYPE_COLNAME,
-                       class_values_colname=CLASS_VALUES_COLNAME,
-                       normalizing_const_colname=NORMALIZING_CONST_COLNAME,
-                       num_classes_colname=NUM_CLASSES_COLNAME,
-                       internal_gpu_config=INTERNAL_GPU_CONFIG,
-                       FLOAT32_SQL_TYPE=FLOAT32_SQL_TYPE)
-        plpy.execute(query)
-
-    def _validate_args(self):
-        validate_module_input_params(
-            self.source_table, self.output_table, self.independent_varname,
-            self.dependent_varname, self.module_name, None,
-            [self.output_summary_table])
-        if self.buffer_size is not None:
-            _assert(self.buffer_size > 0,
-                    "{0}: The buffer size has to be a "
-                    "positive integer or NULL.".format(self.module_name))
-        if self.normalizing_const is not None:
-            _assert(self.normalizing_const > 0,
-                "{0}: The normalizing constant has to be a "
-                "positive integer or NULL.".format(self.module_name))
-
-    def _set_validate_vartypes(self):
-        self.independent_vartype = get_expr_type(self.independent_varname,
-                                                     self.source_table)
-        self.dependent_vartype = get_expr_type(self.dependent_varname,
-                                                   self.source_table)
-        num_of_independent_cols = split_quoted_delimited_str(self.independent_varname)
-        _assert(len(num_of_independent_cols) == 1,
-                "Invalid independent_varname: only one column name is allowed "
-                "as input.")
-        _assert(is_valid_psql_type(self.independent_vartype,
-                                   NUMERIC | ONLY_ARRAY),
-                "Invalid independent variable type, should be an array of "
-                "one of {0}".format(','.join(NUMERIC)))
-        # The dependent variable needs to be either:
-        # 1. NUMERIC, TEXT OR BOOLEAN, which we always one-hot encode
-        # 2. NUMERIC ARRAY, which we assume it is already one-hot encoded, and we
-        #    just cast it the INTEGER ARRAY
-        num_of_dependent_cols = split_quoted_delimited_str(self.dependent_varname)
-        _assert(len(num_of_dependent_cols) == 1,
-                "Invalid dependent_varname: only one column name is allowed "
-                "as input.")
-        _assert((is_valid_psql_type(self.dependent_vartype, NUMERIC | TEXT | BOOLEAN) or
-                 is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY)),
-                """Invalid dependent variable type, should be one of the types in this list:
-                numeric, text, boolean, or numeric array""")
-
-    def get_distinct_dependent_levels(self, table, dependent_varname,
-                                      dependent_vartype):
-        # Refactoring this out into the parent class to ensure include_nulls
-        # is passed in as true for both training and validation tables
-        return get_distinct_col_levels(table, dependent_varname,
-            dependent_vartype, include_nulls=True)
-
-    def _get_buffer_size(self, num_segments):
-        num_rows_in_tbl = plpy.execute("""
-                SELECT count(*) AS cnt FROM {0}
-            """.format(self.source_table))[0]['cnt']
-        buffer_size_calculator = MiniBatchBufferSizeCalculator()
-        indepdent_var_dim = get_product_of_dimensions(self.source_table,
-            self.independent_varname)
-        buffer_size = buffer_size_calculator.calculate_default_buffer_size(
-            self.buffer_size, num_rows_in_tbl, indepdent_var_dim, num_segments)
-        num_buffers = num_segments * ceil((1.0 * num_rows_in_tbl) / buffer_size / num_segments)
-        return int(ceil(num_rows_in_tbl / num_buffers))
-
-class ValidationDataPreprocessorDL(InputDataPreprocessorDL):
-    def __init__(self, schema_madlib, source_table, output_table,
-                 dependent_varname, independent_varname,
-                 training_preprocessor_table, buffer_size, distribution_rules,
-                 **kwargs):
-        """
-            This prepares the variables that are required by
-            InputDataPreprocessorDL.
-        """
-        self.module_name = "validation_preprocessor_dl"
-        self.training_preprocessor_table = training_preprocessor_table
-        summary_table = self._validate_and_process_training_preprocessor_table()
-        num_classes = summary_table[NUM_CLASSES_COLNAME]
-        InputDataPreprocessorDL.__init__(
-            self, schema_madlib, source_table, output_table,
-            dependent_varname, independent_varname, buffer_size,
-            summary_table[NORMALIZING_CONST_COLNAME], num_classes,
-            distribution_rules, self.module_name)
-        # Update value of dependent_levels from training batch summary table.
-        self.dependent_levels = self._get_dependent_levels(
-            summary_table[CLASS_VALUES_COLNAME],
-            summary_table[DEPENDENT_VARTYPE_COLNAME])
-
-    def _get_dependent_levels(self, training_dependent_levels,
-                              training_dependent_vartype):
-        """
-            Return the distinct dependent levels to be considered for
-            one-hot encoding the dependent var. This is inferred from
-            the class_values column in the training_preprocessor_table
-            summary table. Note that class_values in that summary table
-            already has padding in it, so we have to strip it out here
-            in that case.
-            This function also quotes class levels if they are text.
-        """
-        # Validate that dep var type is exactly the same as what was in
-        # trainig_preprocessor_table's input.
-        _assert(self.dependent_vartype == training_dependent_vartype,
-            "{0}: the dependent variable's type in {1} must be {2}.".format(
-                self.module_name, self.source_table,
-                training_dependent_vartype))
-        # training_dependent_levels is the class_values column from the
-        # training batch summary table. This already has the padding with
-        # NULLs in it based on num_classes that was provided to
-        # training_preprocessor_dl(). We have to work our way backwards
-        # to strip out those trailing NULLs from class_values, since
-        # they will anyway get added later in
-        # InputDataPreprocessorDL._set_one_hot_encoding_variables.
-        dependent_levels = strip_trailing_nulls_from_class_values(
-            training_dependent_levels)
-        if training_dependent_levels:
-            dependent_levels_val_data = self.get_distinct_dependent_levels(
-                self.source_table, self.dependent_varname,
-                self.dependent_vartype)
-            unquoted_dependent_levels_val_data = [strip_end_quotes(level, "'")
-                                                  for level in dependent_levels_val_data]
-            # Assert to check if the class values in validation data is a subset
-            # of the class values in training data.
-            _assert(set(unquoted_dependent_levels_val_data).issubset(set(dependent_levels)),
-                    "{0}: the class values in {1} ({2}) should be a "
-                    "subset of class values in {3} ({4})".format(
-                        self.module_name, self.source_table,
-                        unquoted_dependent_levels_val_data,
-                        self.training_preprocessor_table, dependent_levels))
-        if is_psql_char_type(self.dependent_vartype):
-            dependent_levels = [quote_literal(level) if level is not None else level
-                                for level in dependent_levels]
-        return dependent_levels
-
-    def _validate_and_process_training_preprocessor_table(self):
-        """
-            Validate training_preprocessor_table param passed. That and
-            the corresponding summary tables must exist. The summary
-            table must also have columns such as normalizing_const,
-            class_values, num_classes and dependent_vartype in it.
-        """
-        input_tbl_valid(self.training_preprocessor_table, self.module_name)
-        training_summary_table = add_postfix(
-            self.training_preprocessor_table, "_summary")
-        input_tbl_valid(training_summary_table, self.module_name,
-                        error_suffix_str="Please ensure that table '{0}' "
-                                         "has been preprocessed using "
-                                         "training_preprocessor_dl()."
-                                        .format(self.training_preprocessor_table))
-        summary_table = plpy.execute("SELECT * FROM {0} LIMIT 1".format(
-            training_summary_table))[0]
-        _assert(NORMALIZING_CONST_COLNAME in summary_table,
-            "{0}: Expected column {1} in {2}.".format(
-                self.module_name, NORMALIZING_CONST_COLNAME,
-                training_summary_table))
-        _assert(CLASS_VALUES_COLNAME in summary_table,
-            "{0}: Expected column {1} in {2}.".format(
-                self.module_name, CLASS_VALUES_COLNAME,
-                training_summary_table))
-        _assert(NUM_CLASSES_COLNAME in summary_table,
-            "{0}: Expected column {1} in {2}.".format(
-                self.module_name, NUM_CLASSES_COLNAME,
-                training_summary_table))
-        _assert(DEPENDENT_VARTYPE_COLNAME in summary_table,
-            "{0}: Expected column {1} in {2}.".format(
-                self.module_name, DEPENDENT_VARTYPE_COLNAME,
-                training_summary_table))
-        return summary_table
-
-    def validation_preprocessor_dl(self):
-        self.input_preprocessor_dl(order_by_random=False)
-
-class TrainingDataPreprocessorDL(InputDataPreprocessorDL):
-    def __init__(self, schema_madlib, source_table, output_table,
-                 dependent_varname, independent_varname, buffer_size,
-                 normalizing_const, num_classes, distribution_rules,
-                **kwargs):
-        """
-            This prepares the variables that are required by
-            InputDataPreprocessorDL.
-        """
-        self.module_name = "training_preprocessor_dl"
-        InputDataPreprocessorDL.__init__(
-            self, schema_madlib, source_table, output_table,
-            dependent_varname, independent_varname, buffer_size,
-            normalizing_const, num_classes, distribution_rules,
-            self.module_name)
-        # Update default value of dependent_levels in superclass
-        self.dependent_levels = self._get_dependent_levels()
-
-    def _get_dependent_levels(self):
-        """
-            Return the distinct dependent levels to be considered for
-            one-hot encoding the dependent var. class level values of
-            type text are quoted.
-        """
-        if is_valid_psql_type(self.dependent_vartype, NUMERIC | ONLY_ARRAY):
-            dependent_levels = None
-        else:
-            dependent_levels = get_distinct_col_levels(
-                self.source_table, self.dependent_varname,
-                self.dependent_vartype, include_nulls=True)
-        return dependent_levels
-
-    def training_preprocessor_dl(self):
-        self.input_preprocessor_dl(order_by_random=True)
-
-class InputDataPreprocessorDocumentation:
-    @staticmethod
-    def validation_preprocessor_dl_help(schema_madlib, message):
-        method = "validation_preprocessor_dl"
-        summary = """
-        ----------------------------------------------------------------
-                            SUMMARY
-        ----------------------------------------------------------------
-        For Deep Learning based techniques such as Convolutional Neural Nets,
-        the input data is mostly images. These images can be represented as an
-        array of numbers where each element represents a pixel/color intensity.
-        It is standard practice to normalize the image data before use.
-        minibatch_preprocessor() is for general use-cases, but for deep learning
-        based use-cases we provide training_preprocessor_dl() that is
-        light-weight and is specific to image datasets.
-
-        If you want to evaluate the model, a validation dataset has to
-        be prepared. This validation data has to be in the same format
-        as the corresponding batched training data used for training, i.e.,
-        the two datasets must be normalized using the same normalizing
-        constant, and the one-hot encoding of the dependent variable must
-        follow the same convention. validation_preprocessor_dl() can be
-        used to pre-process the validation data. To ensure that the format
-        is similar to the corresponding training data, this function takes
-        the output table name of training_preprocessor_dl() as an input
-        param.
-
-        For more details on function usage:
-        SELECT {schema_madlib}.{method}('usage')
-        """.format(**locals())
-
-        usage = """
-        ---------------------------------------------------------------------------
-                                        USAGE
-        ---------------------------------------------------------------------------
-        SELECT {schema_madlib}.{method}(
-            source_table,          -- TEXT. Name of the table containing input
-                                      data.  Can also be a view.
-            output_table,          -- TEXT. Name of the output table for
-                                      mini-batching.
-            dependent_varname,     -- TEXT. Name of the dependent variable column.
-            independent_varname,   -- TEXT. Name of the independent variable
-                                      column.
-            training_preprocessor_table, -- TEXT. packed training data table.
-            buffer_size            -- INTEGER. Default computed automatically.
-                                      Number of source input rows to pack into a buffer.
-            distribution_rules     -- TEXT. Default: 'all_segments'. Specifies how to
-                                      distribute the 'output_table'. This is important
-                                      for how the fit function will use resources on the
-                                      cluster.  The default 'all_segments' means the
-                                      'output_table' will be distributed to all segments
-                                      in the database cluster.
-        );
-
-
-        ---------------------------------------------------------------------------
-                                        OUTPUT
-        ---------------------------------------------------------------------------
-        The output table produced by validation_preprocessor_dl contains the
-        following columns:
-
-        buffer_id               -- INTEGER.  Unique id for packed table.
-        dependent_varname       -- BYTEA. Packed array of dependent variables.
-        independent_varname     -- BYTEA. Packed array of independent
-                                   variables.
-        dependent_varname       -- TEXT. Shape of the dependent variable buffer.
-        independent_varname     -- TEXT. Shape of the independent variable buffer.
-
-        ---------------------------------------------------------------------------
-        The algorithm also creates a summary table named _summary
-        that has the following columns:
-
-        source_table              -- Source table name.
-        output_table              -- Output table name from preprocessor.
-        dependent_varname         -- Dependent variable values from the original table
-                                     (encoded by one_hot_encode, if specified).
-        independent_varname       -- Independent variable values from the original
-                                     table.
-        dependent_vartype         -- Type of the dependent variable from the
-                                     original table.
-        class_values              -- Class values of the dependent variable
-                                     (‘NULL’(as TEXT type) for non
-                                     categorical vars).
-        buffer_size               -- Buffer size used in preprocessing step.
-        normalizing_const         -- Normalizing constant used for standardizing.
-                                     arrays in independent_varname.
-        num_classes               -- num_classes value passed by user while
-                                     generating training_preprocessor_table.
-        gpu_config                -- List of segment id's the data is distributed
-                                     on depending on the 'distribution_rules' parameter
-                                     specified as input. Set to 'all_segments' if
-                                     'distribution_rules' is specified as 'all_segments'.
-
-        ---------------------------------------------------------------------------
-        """.format(**locals())
-
-        if not message:
-            return summary
-        elif message.lower() in ('usage', 'help', '?'):
-            return usage
-        return """
-            No such option. Use "SELECT {schema_madlib}.{method}()"
-            for help.
-        """.format(**locals())
-
-    @staticmethod
-    def training_preprocessor_dl_help(schema_madlib, message):
-        method = "training_preprocessor_dl"
-        summary = """
-        ----------------------------------------------------------------
-                            SUMMARY
-        ----------------------------------------------------------------
-        For Deep Learning based techniques such as Convolutional Neural Nets,
-        the input data is mostly images. These images can be represented as an
-        array of numbers where each element represents a pixel/color intensity.
-        It is standard practice to normalize the image data before use.
-        minibatch_preprocessor() is for general use-cases, but for deep learning
-        based use-cases we provide training_preprocessor_dl() that is
-        light-weight and is specific to image datasets.
-
-        The normalizing constant is parameterized, and can be specified based
-        on the kind of image data used.
-
-        An optional param named num_classes can be used to specify the length
-        of the one-hot encoded array for the dependent variable. This value if
-        specified must be greater than equal to the total number of distinct
-        class values found in the input table.
-
-        For more details on function usage:
-        SELECT {schema_madlib}.{method}('usage')
-        """.format(**locals())
-
-        usage = """
-        ---------------------------------------------------------------------------
-                                        USAGE
-        ---------------------------------------------------------------------------
-        SELECT {schema_madlib}.{method}(
-            source_table,          -- TEXT. Name of the table containing input
-                                      data.  Can also be a view.
-            output_table,          -- TEXT. Name of the output table for
-                                      mini-batching.
-            dependent_varname,     -- TEXT. Name of the dependent variable column.
-            independent_varname,   -- TEXT. Name of the independent variable
-                                      column.
-            buffer_size            -- INTEGER. Default computed automatically.
-                                      Number of source input rows to pack into a buffer.
-            normalizing_const      -- REAL. Default 1.0. The normalizing constant to
-                                      use for standardizing arrays in independent_varname.
-            num_classes            -- INTEGER. Default NULL. Number of class labels
-                                      to be considered for 1-hot encoding. If NULL,
-                                      the 1-hot encoded array length will be equal to
-                                      the number of distinct class values found in the
-                                      input table.
-            distribution_rules     -- TEXT. Default: 'all_segments'. Specifies how to
-                                      distribute the 'output_table'. This is important
-                                      for how the fit function will use resources on the
-                                      cluster.  The default 'all_segments' means the
-                                      'output_table' will be distributed to all segments
-                                      in the database cluster.
-        );
-
-
-        ---------------------------------------------------------------------------
-                                        OUTPUT
-        ---------------------------------------------------------------------------
-        The output table produced by MiniBatch Preprocessor contains the
-        following columns:
-
-        buffer_id               -- INTEGER.  Unique id for packed table.
-        dependent_varname       -- BYTEA. Packed array of dependent variables.
-        independent_varname     -- BYTEA. Packed array of independent
-                                   variables.
-        dependent_varname       -- TEXT. Shape of the dependent variable buffer.
-        independent_varname     -- TEXT. Shape of the independent variable buffer.
-
-        ---------------------------------------------------------------------------
-        The algorithm also creates a summary table named _summary
-        that has the following columns:
-
-        source_table              -- Source table name.
-        output_table              -- Output table name from preprocessor.
-        dependent_varname         -- Dependent variable values from the original table
-                                     (encoded by one_hot_encode, if specified).
-        independent_varname       -- Independent variable values from the original
-                                     table.
-        dependent_vartype         -- Type of the dependent variable from the
-                                     original table.
-        class_values              -- Class values of the dependent variable
-                                     (‘NULL’(as TEXT type) for non
-                                     categorical vars).
-        buffer_size               -- Buffer size used in preprocessing step.
-        normalizing_const         -- Normalizing constant used for standardizing
-                                     arrays in independent_varname.
-        num_classes               -- num_classes input param passed to function.
-        gpu_config                -- List of segment id's the data is distributed
-                                     on depending on the 'distribution_rules' param
-                                     specified as input. Set to 'all_segments' if
-                                     'distribution_rules' is specified as 'all_segments'.
-
-
-        ---------------------------------------------------------------------------
-        """.format(**locals())
-
-        if not message:
-            return summary
-        elif message.lower() in ('usage', 'help', '?'):
-            return usage
-        return """
-            No such option. Use "SELECT {schema_madlib}.{method}()"
-            for help.
-        """.format(**locals())
diff --git a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in
deleted file mode 100644
index f243417..0000000
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.sql_in
+++ /dev/null
@@ -1,1056 +0,0 @@
-/* ----------------------------------------------------------------------- */
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- * @file input_preprocessor_dl.sql_in
- * @brief Utilities to prepare input image data for use by deep learning modules.
- * @date December 2018
- *
- */
-/* ----------------------------------------------------------------------- */
-
-m4_include(`SQLCommon.m4')
-
-/**
-@addtogroup grp_input_preprocessor_dl
-
-@brief Utilities that prepare input image data for use by deep learning
-modules.
-
-\warning  This MADlib method is still in early stage development.
-Interface and implementation are subject to change. 
-
-
-
-This preprocessor is a utility that prepares image data for use
-by frameworks like Keras and TensorFlow that support mini-batching
-as an optimization option.  The advantage of using mini-batching is that
-it can perform better than stochastic gradient descent
-because it uses more than one training example at a time, typically
-resulting in faster and smoother convergence [1].
-
-Images can be
-represented as an array of numbers
-where each element represents grayscale,
-RGB or other channel values for each
-pixel in the image. It is standard practice to
-normalize the image data before training.
-The normalizing constant in this module is parameterized,
-so it can be set depending on
-the format of image data used.
-
-There are two versions of the preprocessor:
-training_preprocessor_dl() preprocesses input image data to be
-used for training a deep learning model, while
-validation_preprocessor_dl() preprocesses validation
-image data used for model evaluation.
-
-@anchor training_preprocessor_dl
-@par Preprocessor for Training Image Data
-
-
-training_preprocessor_dl(source_table,
-                         output_table,
-                         dependent_varname,
-                         independent_varname,
-                         buffer_size,
-                         normalizing_const,
-                         num_classes,
-                         distribution_rules
-                        )
-
- -\b Arguments -
-
source_table
-
TEXT. Name of the table containing training dataset. - Can also be a view. -
- -
output_table
-
TEXT. Name of the output table from the training preprocessor which - will be used as input to algorithms that support mini-batching. - Note that the arrays packed into the output table are shuffled - and normalized, by dividing each element in the independent variable array - by the optional 'normalizing_const' parameter. For performance reasons, - packed arrays are converted to PostgreSQL bytea format, which is a - variable-length binary string. - - In the case a validation data set is used (see - later on this page), this output table is also used - as an input to the validation preprocessor - so that the validation and training image data are - both preprocessed in an identical manner. -
- -
dependent_varname
-
TEXT. Name of the dependent variable column. - @note The mini-batch preprocessor automatically 1-hot encodes - dependent variables of all types. The exception is numeric array types - (integer and float), where we assume these are already 1-hot encoded, - so these will just be passed through as is. -
- -
independent_varname
-
TEXT. Name of the independent variable column. The column must be - a numeric array type. -
- -
buffer_size (optional)
-
INTEGER, default: computed. Buffer size is the number of rows from - the source table that are packed into one row of the preprocessor output - table. In the case of images, the source table will have one image per row, - and the output table will have multiple images per row. The default value is - computed considering the sizes of the source table and images and the number - of segments in the database cluster. - @note Using the default for 'buffer_size' will produce buffers that are relatively - large, which generally results in the fastest fit() runtime with Keras. Setting a - smaller buffer size may cause the preprocessor to run faster (although this is not - guaranteed, since it depends on database cluster size, data set, and other factors). - But since preprocessing is usually a one-time operation and fit() is called many times, - by default buffer sizes are optimized to make fit() as fast as possible. Note that - specifying a 'buffer_size' does not guarantee that exact value will be used. Actual buffer - size is adjusted to avoid data skew, which adversely impacts fit() runtime. -
- -
normalizing_const (optional)
-
REAL, default: 1.0. The normalizing constant to divide - each value in the 'independent_varname' array by. For example, - you would use 255 for this value if the image data is in the form 0-255. -
- -
num_classes (optional)
-
INTEGER, default: NULL. Number of class labels for 1-hot - encoding. If NULL, the 1-hot encoded array - length will be equal to the number - of distinct class values found in the input table. -
- -
distribution_rules (optional)
-
TEXT, default: 'all_segments'. Specifies how to distribute the - 'output_table'. This is important for how the fit function will use - resources on the cluster. The default 'all_segments' means the - 'output_table' will be distributed to all segments in the database - cluster. - - If you specify 'gpu_segments' then the 'output_table' will be distributed - to all segments that are on hosts that have GPUs attached. This will make - maximum use of GPU resources when training a deep learning model. - - You can also specify the name of a resources table containing the segments - to be used for training. This table must contain a column called 'dbid' that - specifies the segment id from the 'gp_segment_configuration' table [2]. - Refer to the utility function GPU Configuration - for more information on how to - identify segments attached to hosts that are GPU enabled. -
-
- -@anchor validation_preprocessor_dl -@par Preprocessor for Validation Image Data -
-validation_preprocessor_dl(source_table,
-                           output_table,
-                           dependent_varname,
-                           independent_varname,
-                           training_preprocessor_table,
-                           buffer_size,
-                           distribution_rules
-                          )
-
- -\b Arguments -
-
source_table
-
TEXT. Name of the table containing validation dataset. - Can also be a view. -
- -
output_table
-
TEXT. Name of the output table from the validation - preprocessor which will be used as input to algorithms that support mini-batching. - The arrays packed into the output table are - normalized using the same normalizing constant from the - training preprocessor as specified in - the 'training_preprocessor_table' parameter described below. - Validation data is not shuffled. - For performance reasons, - packed arrays are converted to PostgreSQL bytea format, which is a - variable-length binary string. -
- -
dependent_varname
-
TEXT. Name of the dependent variable column. - @note The mini-batch preprocessor automatically 1-hot encodes - dependent variables of all types. The exception is numeric array types - (integer and float), where we assume these are already 1-hot encoded, - so these will just be passed through as is. -
- -
independent_varname
-
TEXT. Name of the independent variable column. The column must be - a numeric array type. -
- -
training_preprocessor_table
-
TEXT. The output table obtained by - running training_preprocessor_dl(). - Validation data is preprocessed in the same way as - training data, i.e., same normalizing constant and dependent - variable class values. -
- -
buffer_size (optional)
-
INTEGER, default: computed. Buffer size is the number of rows from - the source table that are packed into one row of the preprocessor output - table. In the case of images, the source table will have one image per row, - and the output table will have multiple images per row. The default value is - computed considering the sizes of the source table and images and the number - of segments in the database cluster. - @note Using the default for 'buffer_size' will produce buffers that are relatively - large, which generally results in the fastest fit() runtime with Keras. Setting a - smaller buffer size may cause the preprocessor to run faster (although this is not - guaranteed, since it depends on database cluster size, data set, and other factors). - But since preprocessing is usually a one-time operation and fit() is called many times, - by default buffer sizes are optimized to make fit() as fast as possible. Note that - specifying a 'buffer_size' does not guarantee that exact value will be used. Actual buffer - size is adjusted to avoid data skew, which adversely impacts fit() runtime. -
- -
distribution_rules (optional)
-
TEXT, default: 'all_segments'. Specifies how to distribute the - 'output_table'. This is important for how the fit function will use - resources on the cluster. The default 'all_segments' means the - 'output_table' will be distributed to all segments in the database - cluster. - - If you specify 'gpu_segments' then the 'output_table' will be distributed - to all segments that are on hosts that have GPUs attached. This will make - maximum use of GPU resources when training a deep learning model. - - You can also specify the name of a resources table containing the segments - to be used for training. This table must contain a column called 'dbid' that - specifies the segment id from the 'gp_segment_configuration' table [2]. - Refer to the utility function GPU Configuration - for more information on how to - identify segments attached to hosts that are GPU enabled. -
- -
- -@anchor output -@par Output Tables -
- The output tables produced by both training_preprocessor_dl() and - validation_preprocessor_dl() contain the following columns: - - - - - - - - - - - - - - - - - - - - - -
independent_varBYTEA. Packed array of independent variables in PostgreSQL bytea format. - Arrays of independent variables packed into the output table are - normalized by dividing each element in the independent variable array by the - optional 'normalizing_const' parameter. Training data is shuffled, but - validation data is not. -
dependent_varBYTEA. Packed array of dependent variables in PostgreSQL bytea format. - The dependent variable is always one-hot encoded as an - integer array. For now, we are assuming that - input_preprocessor_dl() will be used - only for classification problems using deep learning. So - the dependent variable is one-hot encoded, unless it's already a - numeric array in which case we assume it's already one-hot - encoded and just cast it to an integer array. -
independent_var_shapeINTEGER[]. Shape of the independent variable array after preprocessing. - The first element is the number of images packed per row, and subsequent - elements will depend on how the image is described (e.g., channels first or last). -
dependent_var_shapeINTEGER[]. Shape of the dependent variable array after preprocessing. - The first element is the number of images packed per row, and the second - element is the number of class values. -
buffer_idINTEGER. Unique id for each row in the packed table. -
- -A summary table named \_summary is also created, which -has the following columns (the columns are the same for -both validation_preprocessor_dl() and training_preprocessor_dl() ): - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
source_tableName of the source table.
output_tableName of output table generated by preprocessor.
dependent_varnameDependent variable from the source table.
independent_varnameIndependent variable from the source table.
dependent_vartypeType of the dependent variable from the source table.
class_valuesThe dependent level values that one-hot encoding maps to.
buffer_sizeBuffer size used in preprocessing step.
normalizing_constThe value used to normalize the input image data.
num_classesNumber of dependent levels the one-hot encoding is created - for. NULLs are padded at the end if the number of distinct class - levels found in the input data is less than the 'num_classes' parameter - specified in training_preprocessor_dl().
distribution_rulesThis is the list of segment id's in the form of 'dbid' - describing how the 'output_table' is distributed, - as per the 'distribution_rules' input parameter. - If the 'distribution_rules' parameter is set to 'all_segments', then - this will also be set to 'all_segments'.
__internal_gpu_config__For internal use. (Note: this is the list of segment id's - where data is distributed in the form of 'content' id, which - is different from 'dbid' [2].)
- -@anchor example -@par Examples --# Create an artificial 2x2 resolution color image data set with 3 possible classifications. -The RGB values are per-pixel arrays: -
-DROP TABLE IF EXISTS image_data;
-CREATE TABLE image_data AS (
-    SELECT ARRAY[
-        ARRAY[
-            ARRAY[(random() * 256)::integer, -- pixel (1,1)
-                (random() * 256)::integer,
-                (random() * 256)::integer],
-            ARRAY[(random() * 256)::integer, -- pixel (2,1)
-                (random() * 256)::integer,
-                (random() * 256)::integer]
-        ],
-        ARRAY[
-            ARRAY[(random() * 256)::integer, -- pixel (1,2)
-                (random() * 256)::integer,
-                (random() * 256)::integer],
-            ARRAY[(random() * 256)::integer, -- pixel (2,1)
-                (random() * 256)::integer,
-                (random() * 256)::integer]
-        ]
-    ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species
-    FROM generate_series(1, 52)
-);
-SELECT * FROM image_data;
-
-
-                             rgb                              | species
---------------------------------------------------------------+---------
- {{{124,198,44},{91,47,130}},{{24,175,69},{196,189,166}}}     | dog
- {{{111,202,129},{198,249,254}},{{141,37,88},{187,167,113}}}  | dog
- {{{235,53,39},{145,167,209}},{{197,147,222},{55,218,53}}}    | dog
- {{{231,48,125},{248,233,151}},{{63,125,230},{33,24,70}}}     | dog
- {{{92,146,121},{163,241,110}},{{75,88,72},{218,90,12}}}      | bird
- {{{88,114,59},{202,211,152}},{{92,76,58},{77,186,134}}}      | dog
- {{{2,96,255},{14,48,19}},{{240,55,115},{137,255,245}}}       | dog
- {{{165,122,98},{16,115,240}},{{4,106,116},{108,242,210}}}    | dog
- {{{155,207,101},{214,167,24}},{{118,240,228},{199,230,21}}}  | dog
- {{{94,212,15},{48,66,170}},{{255,167,128},{166,191,246}}}    | dog
- {{{169,69,131},{16,98,225}},{{228,113,17},{38,27,17}}}       | bird
- {{{156,183,139},{146,77,46}},{{80,202,230},{146,84,239}}}    | dog
- {{{190,210,147},{227,31,66}},{{229,251,84},{51,118,240}}}    | bird
- {{{253,175,200},{237,151,107}},{{207,56,162},{133,39,35}}}   | cat
- {{{146,185,108},{14,10,105}},{{188,210,86},{83,61,36}}}      | dog
- {{{223,169,177},{3,200,250}},{{112,91,16},{193,32,151}}}     | cat
- {{{249,145,240},{144,153,58}},{{131,156,230},{56,50,75}}}    | dog
- {{{212,186,229},{52,251,197}},{{230,121,201},{35,215,119}}}  | cat
- {{{234,94,23},{114,196,94}},{{242,249,90},{223,24,109}}}     | bird
- {{{111,36,145},{77,135,123}},{{171,158,237},{111,252,222}}}  | dog
- {{{90,74,240},{231,133,95}},{{11,21,173},{146,144,88}}}      | cat
- {{{170,52,237},{13,114,71}},{{87,99,46},{220,194,56}}}       | bird
- {{{8,17,92},{64,2,203}},{{10,131,145},{4,129,30}}}           | cat
- {{{217,218,207},{74,68,186}},{{127,107,76},{38,60,16}}}      | bird
- {{{193,34,83},{203,99,58}},{{251,224,50},{228,118,113}}}     | dog
- {{{146,218,155},{32,159,243}},{{146,218,189},{101,114,25}}}  | bird
- {{{179,160,74},{204,81,246}},{{50,189,39},{60,42,185}}}      | cat
- {{{13,82,174},{198,151,84}},{{65,249,100},{179,234,104}}}    | cat
- {{{162,190,124},{184,66,138}},{{10,240,80},{161,68,145}}}    | dog
- {{{164,144,199},{53,42,111}},{{122,174,128},{220,143,100}}}  | cat
- {{{160,138,104},{177,86,3}},{{104,226,149},{181,16,229}}}    | dog
- {{{246,119,211},{229,249,119}},{{117,192,172},{159,47,38}}}  | cat
- {{{175,1,220},{18,78,124}},{{156,181,45},{242,185,148}}}     | bird
- {{{50,113,246},{101,213,180}},{{56,103,151},{87,169,124}}}   | cat
- {{{73,109,147},{22,81,197}},{{135,71,42},{91,251,98}}}       | bird
- {{{206,61,255},{25,151,211}},{{211,124,7},{206,64,237}}}     | cat
- {{{201,71,34},{182,142,43}},{{198,172,171},{230,1,23}}}      | bird
- {{{142,158,2},{223,45,205}},{{118,177,223},{232,178,141}}}   | cat
- {{{86,190,128},{195,172,14}},{{97,173,237},{142,123,99}}}    | cat
- {{{26,72,148},{79,226,156}},{{96,62,220},{99,9,230}}}        | bird
- {{{154,234,103},{184,18,65}},{{146,225,139},{214,156,10}}}   | cat
- {{{244,169,103},{218,143,2}},{{196,246,186},{214,55,76}}}    | bird
- {{{20,226,7},{96,153,200}},{{130,236,147},{229,38,142}}}     | bird
- {{{172,102,107},{50,11,109}},{{145,9,123},{193,28,107}}}     | bird
- {{{143,243,247},{132,104,137}},{{94,3,169},{253,246,59}}}    | bird
- {{{78,74,228},{51,200,218}},{{170,155,190},{164,18,51}}}     | dog
- {{{163,226,161},{56,182,239}},{{129,154,35},{73,116,205}}}   | bird
- {{{74,243,3},{172,182,149}},{{101,34,163},{111,138,95}}}     | cat
- {{{224,178,126},{4,61,93}},{{174,238,96},{118,232,208}}}     | bird
- {{{55,236,249},{7,189,242}},{{151,173,130},{49,232,5}}}      | bird
- {{{9,16,30},{128,32,85}},{{108,25,91},{41,11,243}}}          | bird
- {{{141,35,191},{146,240,141}},{{207,239,166},{102,194,121}}} | bird
-(52 rows)
-
--# Run the preprocessor for training image data: -
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data',         -- Source table
-                                        'image_data_packed',  -- Output table
-                                        'species',            -- Dependent variable
-                                        'rgb',                -- Independent variable
-                                        NULL,                 -- Buffer size
-                                        255                   -- Normalizing constant
-                                        );
-
-For small datasets like in this example, buffer size is mainly determined -by the number of segments in the database. For a Greenplum database with 2 segments, -there will be 2 rows with a buffer size of 26. For PostgresSQL, there would -be only one row with a buffer size of 52 since it is a single node database. -For larger data sets, other factors go into computing buffers size besides -number of segments. -Here is the packed output table of training data for our simple example: -
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM image_data_packed ORDER BY buffer_id;
-
-
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,2,2,3}            | {26,3}              |         0
- {26,2,2,3}            | {26,3}              |         1
-(2 rows)
-
-Review the output summary table: -
-\\x on
-SELECT * FROM image_data_packed_summary;
-
-
--[ RECORD 1 ]-----------+------------------
-source_table            | image_data
-output_table            | image_data_packed
-dependent_varname       | species
-independent_varname     | rgb
-dependent_vartype       | text
-class_values            | {bird,cat,dog}
-buffer_size             | 26
-normalizing_const       | 255
-num_classes             | 3
-distribution_rules      | all_segments
-__internal_gpu_config__ | all_segments
-
- --# Run the preprocessor for the validation dataset. -In this example, we use the same images for -validation to demonstrate, but normally validation data -is different than training data: -
-DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;
-SELECT madlib.validation_preprocessor_dl(
-      'image_data',             -- Source table
-      'val_image_data_packed',  -- Output table
-      'species',                -- Dependent variable
-      'rgb',                    -- Independent variable
-      'image_data_packed',      -- From training preprocessor step
-      NULL                      -- Buffer size
-      );
-
-We could choose to use a different buffer size compared to the -training_preprocessor_dl run (but generally don't need to). -Other parameters such as num_classes and -normalizing_const that were passed to training_preprocessor_dl are -automatically inferred using the image_data_packed param that is passed. -Here is the packed output table of validation data for our simple example: -
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM val_image_data_packed ORDER BY buffer_id;
-
-
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,2,2,3}            | {26,3}              |         0
- {26,2,2,3}            | {26,3}              |         1
-(2 rows)
-
-Review the output summary table: -
-\\x on
-SELECT * FROM val_image_data_packed_summary;
-
-
--[ RECORD 1 ]-----------+----------------------
-source_table            | image_data
-output_table            | val_image_data_packed
-dependent_varname       | species
-independent_varname     | rgb
-dependent_vartype       | text
-class_values            | {bird,cat,dog}
-buffer_size             | 26
-normalizing_const       | 255
-num_classes             | 3
-distribution_rules      | all_segments
-__internal_gpu_config__ | all_segments
-
- --# Load data in another format. Create an artificial 2x2 resolution color image -data set with 3 possible classifications. The RGB values are unrolled into a flat array: -
-DROP TABLE IF EXISTS image_data;
-CREATE TABLE image_data AS (
-SELECT ARRAY[
-        (random() * 256)::integer, -- R values
-        (random() * 256)::integer,
-        (random() * 256)::integer,
-        (random() * 256)::integer,
-        (random() * 256)::integer, -- G values
-        (random() * 256)::integer,
-        (random() * 256)::integer,
-        (random() * 256)::integer,
-        (random() * 256)::integer, -- B values
-        (random() * 256)::integer,
-        (random() * 256)::integer,
-        (random() * 256)::integer
-    ] as rgb, ('{cat,dog,bird}'::text[])[ceil(random()*3)] as species
-FROM generate_series(1, 52)
-);
-SELECT * FROM image_data;
-
-
-                       rgb                        | species
---------------------------------------------------+---------
- {26,150,191,113,235,57,145,143,44,145,85,25}     | dog
- {240,43,225,15,220,136,186,209,49,130,55,111}    | bird
- {25,191,37,77,193,62,249,228,97,33,81,7}         | cat
- {141,223,46,195,201,19,207,78,160,130,157,89}    | cat
- {39,249,168,164,223,193,99,4,14,37,66,7}         | cat
- {159,250,127,44,151,254,11,211,247,137,79,233}   | cat
- {19,230,76,253,42,175,230,143,184,133,27,215}    | cat
- {199,224,144,5,64,19,200,186,109,218,108,70}     | bird
- {148,136,4,41,185,104,203,253,113,151,166,76}    | bird
- {230,132,114,213,210,139,91,199,240,142,203,75}  | bird
- {166,188,96,217,135,70,93,249,27,47,132,118}     | bird
- {118,120,222,236,110,83,240,47,19,206,222,51}    | bird
- {230,3,26,47,93,144,167,59,123,21,142,107}       | cat
- {250,224,62,136,112,142,88,187,24,1,168,216}     | bird
- {52,144,231,12,76,1,162,11,114,141,69,3}         | cat
- {166,172,246,169,200,102,62,57,239,75,165,88}    | dog
- {151,50,112,227,199,97,47,4,43,123,116,133}      | bird
- {39,185,96,127,80,248,177,191,218,120,32,9}      | dog
- {25,172,34,34,40,109,166,23,60,216,246,54}       | bird
- {163,39,89,170,95,230,137,141,169,82,159,121}    | dog
- {131,143,183,138,151,90,177,240,4,16,214,141}    | dog
- {99,233,100,9,159,140,30,202,29,169,120,62}      | bird
- {99,162,69,10,204,169,219,20,106,170,111,16}     | bird
- {16,246,27,32,187,226,0,75,231,64,94,175}        | bird
- {25,135,244,101,50,4,91,77,36,22,47,37}          | dog
- {22,101,191,197,96,138,78,198,155,138,193,51}    | bird
- {236,22,110,30,181,20,218,21,236,97,91,73}       | dog
- {160,57,34,212,239,197,233,174,164,97,88,153}    | cat
- {226,170,192,123,242,224,190,51,163,192,91,105}  | bird
- {149,174,12,72,112,1,37,153,118,201,79,121}      | bird
- {34,250,232,222,218,221,234,201,138,66,186,58}   | bird
- {162,55,85,159,247,234,77,3,50,189,4,87}         | dog
- {122,32,164,243,0,198,237,232,164,199,197,142}   | dog
- {80,209,75,138,169,236,193,254,140,184,232,217}  | bird
- {112,148,114,137,13,107,105,75,243,218,218,75}   | dog
- {241,76,61,202,76,112,90,51,125,166,52,30}       | bird
- {75,132,239,207,49,224,250,19,238,214,154,169}   | dog
- {203,43,222,58,231,5,243,71,131,67,63,52}        | cat
- {229,12,133,142,179,80,185,145,138,160,149,125}  | bird
- {64,251,61,153,13,100,145,181,8,112,118,107}     | dog
- {128,223,60,248,126,124,243,188,20,0,31,166}     | bird
- {39,22,43,146,138,174,33,65,56,184,155,234}      | dog
- {177,247,133,154,159,37,148,30,81,43,29,92}      | bird
- {56,127,199,118,105,120,109,239,18,12,20,166}    | cat
- {101,209,72,193,207,91,166,27,88,209,203,62}     | dog
- {131,195,122,90,18,178,217,217,40,66,81,149}     | cat
- {203,137,103,17,60,251,152,64,36,81,168,239}     | cat
- {239,97,10,20,194,32,121,129,228,217,11,50}      | dog
- {117,4,193,192,223,176,33,232,196,226,8,61}      | dog
- {162,21,190,223,120,170,245,230,200,170,250,163} | bird
- {32,67,65,195,2,39,198,28,86,35,172,254}         | dog
- {39,19,236,146,87,140,203,121,96,187,62,73}      | dog
-(52 rows)
-
- --# Run the preprocessor for training image data: -
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data',         -- Source table
-                                        'image_data_packed',  -- Output table
-                                        'species',            -- Dependent variable
-                                        'rgb',                -- Independent variable
-                                        NULL,                 -- Buffer size
-                                        255                   -- Normalizing constant
-                                        );
-
-Here is a sample of the packed output table: -
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM image_data_packed ORDER BY buffer_id;
-
-
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,12}               | {26,3}              |         0
- {26,12}               | {26,3}              |         1
-(2 rows)
-
- --# Run the preprocessor for the validation dataset. -In this example, we use the same images for -validation to demonstrate, but normally validation data -is different than training data: -
-DROP TABLE IF EXISTS val_image_data_packed, val_image_data_packed_summary;
-SELECT madlib.validation_preprocessor_dl(
-    'image_data',             -- Source table
-    'val_image_data_packed',  -- Output table
-    'species',                -- Dependent variable
-    'rgb',                    -- Independent variable
-    'image_data_packed',      -- From training preprocessor step
-    NULL                      -- Buffer size
-    );
-
-Here is a sample of the packed output summary table: -
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM val_image_data_packed ORDER BY buffer_id;
-
-
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,12}               | {26,3}              |         0
- {26,12}               | {26,3}              |         1
-(2 rows)
-
- --# Generally the default buffer size will work well, -but if you have occasion to change it: -
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data',         -- Source table
-                                       'image_data_packed',  -- Output table
-                                       'species',            -- Dependent variable
-                                       'rgb',                -- Independent variable
-                                        10,                   -- Buffer size
-                                        255                   -- Normalizing constant
-                                        );
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM image_data_packed ORDER BY buffer_id;
-
-
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {8,12}                | {8,3}               |         0
- {9,12}                | {9,3}               |         1
- {9,12}                | {9,3}               |         2
- {9,12}                | {9,3}               |         3
- {9,12}                | {9,3}               |         4
- {8,12}                | {8,3}               |         5
-(6 rows)
-
-Review the output summary table: -
-\\x on
-SELECT * FROM image_data_packed_summary;
-
-
--[ RECORD 1 ]-----------+------------------
-source_table            | image_data
-output_table            | image_data_packed
-dependent_varname       | species
-independent_varname     | rgb
-dependent_vartype       | text
-class_values            | {bird,cat,dog}
-buffer_size             | 10
-normalizing_const       | 255
-num_classes             | 3
-distribution_rules      | all_segments
-__internal_gpu_config__ | all_segments
-
- --# Run the preprocessor for image data with num_classes greater than 3 (distinct class values found in table): -
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data',         -- Source table
-                                        'image_data_packed',  -- Output table
-                                        'species',            -- Dependent variable
-                                        'rgb',                -- Independent variable
-                                        NULL,                 -- Buffer size
-                                        255,                  -- Normalizing constant
-                                        5                     -- Number of desired class values
-                                        );
-
-Here is a sample of the packed output table with the padded 1-hot vector: -
-SELECT independent_var_shape, dependent_var_shape, buffer_id FROM image_data_packed ORDER BY buffer_id;
-
-
- independent_var_shape | dependent_var_shape | buffer_id
------------------------+---------------------+-----------
- {26,12}               | {26,5}              |         0
- {26,12}               | {26,5}              |         1
-(2 rows)
-
-Review the output summary table: -
-\\x on
-SELECT * FROM image_data_packed_summary;
-
-
--[ RECORD 1 ]-----------+-------------------------
-source_table            | image_data
-output_table            | image_data_packed
-dependent_varname       | species
-independent_varname     | rgb
-dependent_vartype       | text
-class_values            | {bird,cat,dog,NULL,NULL}
-buffer_size             | 26
-normalizing_const       | 255
-num_classes             | 5
-distribution_rules      | all_segments
-__internal_gpu_config__ | all_segments
-
- --# Using distribution rules to specify how to distribute -the 'output_table'. This is important for how the fit function -will use resources on the cluster. To distribute to all segments -on hosts with GPUs attached: -
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data',          -- Source table
-                                        'image_data_packed',  -- Output table
-                                        'species',            -- Dependent variable
-                                        'rgb',                -- Independent variable
-                                        NULL,                 -- Buffer size
-                                        255,                  -- Normalizing constant
-                                        NULL,                 -- Number of classes
-                                        'gpu_segments'        -- Distribution rules
-                                        );
-\\x on
-SELECT * FROM image_data_packed_summary;
-
-
--[ RECORD 1 ]-----------+------------------
-source_table            | image_data
-output_table            | image_data_packed
-dependent_varname       | species
-independent_varname     | rgb
-dependent_vartype       | text
-class_values            | {bird,cat,dog}
-buffer_size             | 26
-normalizing_const       | 255
-num_classes             | 3
-distribution_rules      | {2,3,4,5}
-__internal_gpu_config__ | {0,1,2,3}
-
-To distribute to only specified segments, create a -distribution table with a column called 'dbid' that -lists the segments you want: -
-DROP TABLE IF EXISTS segments_to_use;
-CREATE TABLE segments_to_use(
-    dbid INTEGER,
-    hostname TEXT
-);
-INSERT INTO segments_to_use VALUES
-(2, 'hostname-01'),
-(3, 'hostname-01');
-DROP TABLE IF EXISTS image_data_packed, image_data_packed_summary;
-SELECT madlib.training_preprocessor_dl('image_data',          -- Source table
-                                        'image_data_packed',  -- Output table
-                                        'species',            -- Dependent variable
-                                        'rgb',                -- Independent variable
-                                        NULL,                 -- Buffer size
-                                        255,                  -- Normalizing constant
-                                        NULL,                 -- Number of classes
-                                        'segments_to_use'     -- Distribution rules
-                                        );
-\\x on
-SELECT * FROM image_data_packed_summary;
-
-
--[ RECORD 1 ]-----------+------------------
-source_table            | image_data
-output_table            | image_data_packed
-dependent_varname       | species
-independent_varname     | rgb
-dependent_vartype       | text
-class_values            | {bird,cat,dog}
-buffer_size             | 26
-normalizing_const       | 255
-num_classes             | 3
-distribution_rules      | {2,3}
-__internal_gpu_config__ | {0,1}
-
- -@anchor references -@par References - -[1] "Neural Networks for Machine Learning", Lectures 6a and 6b on mini-batch gradient descent, -Geoffrey Hinton with Nitish Srivastava and Kevin Swersky, http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf - -[2] Greenplum 'gp_segment_configuration' table https://gpdb.docs.pivotal.io/latest/ref_guide/system_catalogs/gp_segment_configuration.html - -@anchor related -@par Related Topics - -training_preprocessor_dl() - -validation_preprocessor_dl() - -gpu_configuration() - -*/ - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl( - source_table VARCHAR, - output_table VARCHAR, - dependent_varname VARCHAR, - independent_varname VARCHAR, - training_preprocessor_table VARCHAR, - buffer_size INTEGER, - distribution_rules TEXT -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(deep_learning, input_data_preprocessor) - from utilities.control import MinWarning - with AOControl(False): - with MinWarning('error'): - validation_preprocessor_obj = input_data_preprocessor.ValidationDataPreprocessorDL(**globals()) - validation_preprocessor_obj.validation_preprocessor_dl() -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl( - source_table VARCHAR, - output_table VARCHAR, - dependent_varname VARCHAR, - independent_varname VARCHAR, - training_preprocessor_table VARCHAR, - buffer_size INTEGER -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.validation_preprocessor_dl($1, $2, $3, $4, $5, $6, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl( - source_table VARCHAR, - output_table VARCHAR, - dependent_varname VARCHAR, - independent_varname VARCHAR, - training_preprocessor_table VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.validation_preprocessor_dl($1, $2, $3, $4, $5, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl( - message VARCHAR -) RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(deep_learning, input_data_preprocessor) - return input_data_preprocessor.InputDataPreprocessorDocumentation.validation_preprocessor_dl_help(schema_madlib, message) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.validation_preprocessor_dl() -RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(deep_learning, input_data_preprocessor) - return input_data_preprocessor.InputDataPreprocessorDocumentation.validation_preprocessor_dl_help(schema_madlib, '') -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -------------------------------------------------------------------------------- - - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl( - source_table VARCHAR, - output_table VARCHAR, - dependent_varname VARCHAR, - independent_varname VARCHAR, - buffer_size INTEGER, - normalizing_const REAL, - num_classes INTEGER, - distribution_rules TEXT -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(deep_learning, input_data_preprocessor) - from utilities.control import MinWarning - with AOControl(False): - with MinWarning('error'): - training_preprocessor_obj = input_data_preprocessor.TrainingDataPreprocessorDL(**globals()) - training_preprocessor_obj.training_preprocessor_dl() -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl( - source_table VARCHAR, - output_table VARCHAR, - dependent_varname VARCHAR, - independent_varname VARCHAR, - buffer_size INTEGER, - normalizing_const REAL, - num_classes INTEGER -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, $6, $7, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl( - source_table VARCHAR, - output_table VARCHAR, - dependent_varname VARCHAR, - independent_varname VARCHAR, - buffer_size INTEGER, - normalizing_const REAL -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, $6, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl( - source_table VARCHAR, - output_table VARCHAR, - dependent_varname VARCHAR, - independent_varname VARCHAR, - buffer_size INTEGER -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, $5, 1.0, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl( - source_table VARCHAR, - output_table VARCHAR, - dependent_varname VARCHAR, - independent_varname VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.training_preprocessor_dl($1, $2, $3, $4, NULL, 1.0, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl( - message VARCHAR -) RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(deep_learning, input_data_preprocessor) - return input_data_preprocessor.InputDataPreprocessorDocumentation.training_preprocessor_dl_help(schema_madlib, message) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.training_preprocessor_dl() -RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(deep_learning, input_data_preprocessor) - return input_data_preprocessor.InputDataPreprocessorDocumentation.training_preprocessor_dl_help(schema_madlib, '') -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - --- aggregation for independent var -DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(REAL[]); -CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(REAL[]) ( - SFUNC = array_cat, - PREFUNC = array_cat, - STYPE = REAL[] - ); - --- aggregation for dependent var -DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.agg_array_concat(SMALLINT[]); -CREATE AGGREGATE MADLIB_SCHEMA.agg_array_concat(SMALLINT[]) ( - SFUNC = array_cat, - PREFUNC = array_cat, - STYPE = SMALLINT[] - ); - -CREATE FUNCTION MADLIB_SCHEMA.convert_array_to_bytea(var REAL[]) -RETURNS BYTEA -AS -$$ -import numpy as np - -return np.array(var, dtype=np.float32).tobytes() -$$ LANGUAGE plpythonu; - -CREATE FUNCTION MADLIB_SCHEMA.convert_array_to_bytea(var SMALLINT[]) -RETURNS BYTEA -AS -$$ -import numpy as np - -return np.array(var, dtype=np.int16).tobytes() -$$ LANGUAGE plpythonu; - - -CREATE FUNCTION MADLIB_SCHEMA.convert_bytea_to_real_array(var BYTEA) -RETURNS REAL[] -AS -$$ -import numpy as np - -return np.frombuffer(var, dtype=np.float32) -$$ LANGUAGE plpythonu; - - -CREATE FUNCTION MADLIB_SCHEMA.convert_bytea_to_smallint_array(var BYTEA) -RETURNS SMALLINT[] -AS -$$ -import numpy as np - -return np.frombuffer(var, dtype=np.int16) -$$ LANGUAGE plpythonu; - diff --git a/src/ports/postgres/modules/deep_learning/keras_model_arch_table.py_in b/src/ports/postgres/modules/deep_learning/keras_model_arch_table.py_in deleted file mode 100644 index 0d6fc7c..0000000 --- a/src/ports/postgres/modules/deep_learning/keras_model_arch_table.py_in +++ /dev/null @@ -1,231 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -@file keras_model_arch_table.py_in - -@brief keras model arch table management helper functions - -@namespace keras_model_arch_table -""" - -from internal.db_utils import quote_literal -import plpy -from utilities.control import MinWarning -from utilities.utilities import get_col_name_type_sql_string -from utilities.utilities import unique_string -from utilities.validate_args import columns_missing_from_table -from utilities.validate_args import input_tbl_valid -from utilities.validate_args import quote_ident -from utilities.validate_args import table_exists - -class ModelArchSchema: - """Expected format of keras_model_arch_table. - Example uses: - - from utilities.validate_args import columns_missing_from_table - from keras_model_arch_table import Format - - # Validate names in cols list against actual table - missing_cols = columns_missing_from_table('my_arch_table', Format.col_names) - - # Get model arch from keras model arch table, without hard coding column names - sql = "SELECT {arch} FROM {table} WHERE {id} = {my_id}" - .format(arch=Format.model_arch, - table='my_arch_table', - id=Format.model_id, - my_id=1) - arch = plpy.execute(sql)[0] - - """ - col_names = ('model_id', 'model_arch', 'model_weights', 'name', 'description', - '__internal_madlib_id__') - col_types = ('SERIAL PRIMARY KEY', 'JSON', 'bytea', 'TEXT', 'TEXT', 'TEXT') - (MODEL_ID, MODEL_ARCH, MODEL_WEIGHTS, NAME, DESCRIPTION, - __INTERNAL_MADLIB_ID__) = col_names - -@MinWarning("error") -def load_keras_model(keras_model_arch_table, model_arch, model_weights, - name, description, **kwargs): - model_arch_table = quote_ident(keras_model_arch_table) - if not table_exists(model_arch_table): - col_defs = get_col_name_type_sql_string(ModelArchSchema.col_names, - ModelArchSchema.col_types) - - sql = "CREATE TABLE {model_arch_table} ({col_defs})" \ - .format(**locals()) - - plpy.execute(sql, 0) - plpy.info("Keras Model Arch: Created new keras model architecture table {0}." \ - .format(model_arch_table)) - else: - missing_cols = columns_missing_from_table(model_arch_table, - ModelArchSchema.col_names) - if len(missing_cols) > 0: - plpy.error("Keras Model Arch: Invalid keras model architecture table {0}," - " missing columns: {1}".format(model_arch_table, - missing_cols)) - - unique_str = unique_string(prefix_has_temp=False) - insert_query = plpy.prepare("INSERT INTO {model_arch_table} " - "VALUES(DEFAULT, $1, $2, $3, $4, $5);".format(**locals()), - ModelArchSchema.col_types[1:]) - insert_res = plpy.execute(insert_query,[model_arch, model_weights, name, description, - unique_str], 0) - - select_query = """SELECT {model_id_col}, {model_arch_col} FROM {model_arch_table} - WHERE {internal_id_col} = '{unique_str}'""".format( - model_id_col=ModelArchSchema.MODEL_ID, - model_arch_col=ModelArchSchema.MODEL_ARCH, - model_arch_table=model_arch_table, - internal_id_col=ModelArchSchema.__INTERNAL_MADLIB_ID__, - unique_str=unique_str) - select_res = plpy.execute(select_query,1) - - plpy.info("Keras Model Arch: Added model id {0} to {1} table". - format(select_res[0][ModelArchSchema.MODEL_ID], model_arch_table)) - -@MinWarning("error") -def delete_keras_model(keras_model_arch_table, model_id, **kwargs): - model_arch_table = quote_ident(keras_model_arch_table) - input_tbl_valid(model_arch_table, "Keras Model Arch") - - missing_cols = columns_missing_from_table(model_arch_table, ModelArchSchema.col_names) - if len(missing_cols) > 0: - plpy.error("Keras Model Arch: Invalid keras model architecture table {0}," - " missing columns: {1}".format(model_arch_table, missing_cols)) - - sql = """ - DELETE FROM {model_arch_table} WHERE {model_id_col}={model_id} - """.format(model_arch_table=model_arch_table, model_id_col=ModelArchSchema.MODEL_ID, - model_id=model_id) - res = plpy.execute(sql, 0) - - if res.nrows() > 0: - plpy.info("Keras Model Arch: Model id {0} has been deleted from {1}.". - format(model_id, model_arch_table)) - else: - plpy.error("Keras Model Arch: Model id {0} not found".format(model_id)) - - sql = "SELECT {0} FROM {1}".format(ModelArchSchema.MODEL_ID, model_arch_table) - res = plpy.execute(sql, 0) - if not res: - plpy.info("Keras Model Arch: Dropping empty keras model architecture "\ - "table {model_arch_table}".format(model_arch_table=model_arch_table)) - sql = "DROP TABLE {0}".format(model_arch_table) - plpy.execute(sql, 0) - -class KerasModelArchDocumentation: - @staticmethod - def _returnHelpMsg(schema_madlib, message, summary, usage, method): - if not message: - return summary - elif message.lower() in ('usage', 'help', '?'): - return usage - return """ - No such option. Use "SELECT {schema_madlib}.{method}()" - for help. - """.format(**locals()) - - @staticmethod - def load_keras_model_help(schema_madlib, message): - method = "load_keras_model" - summary = """ - ---------------------------------------------------------------- - SUMMARY - ---------------------------------------------------------------- - The architecture of the model to be used in madlib_keras_train() - function must be stored in a table, the details of which must be - provided as parameters to the madlib_keras_train module. This is - a helper function to help users insert JSON blobs of Keras model - architectures into a table. - If the output table already exists, the model_arch specified will - be added as a new row into the table. The output table could thus - act as a repository of Keras model architectures. - - For more details on function usage: - SELECT {schema_madlib}.{method}('usage') - """.format(**locals()) - - usage = """ - --------------------------------------------------------------------------- - USAGE - --------------------------------------------------------------------------- - SELECT {schema_madlib}.{method}( - keras_model_arch_table, -- Output table to load keras model arch. - model_arch, -- JSON of the model architecture to insert. - model_weights, -- Model weights to load as a PostgreSQL - binary data type. - name, -- Free text string to identify a name - description -- Free text string to provide a description - ); - - - --------------------------------------------------------------------------- - OUTPUT - --------------------------------------------------------------------------- - The output table produced by load_keras_model contains the following columns: - - 'model_id' -- SERIAL PRIMARY KEY. Model ID. - 'model_arch' -- JSON. JSON blob of the model architecture. - 'model_weights' -- bytea. weights of the model for warm start. - '__internal_madlib_id__' -- TEXT. Unique id for model arch. - - """.format(**locals()) - - return KerasModelArchDocumentation._returnHelpMsg( - schema_madlib, message, summary, usage, method) -# --------------------------------------------------------------------- - - @staticmethod - def delete_keras_model_help(schema_madlib, message): - method = "delete_keras_model" - summary = """ - ---------------------------------------------------------------- - SUMMARY - ---------------------------------------------------------------- - Delete the model architecture corresponding to the provided model_id - from the model architecture repository table (keras_model_arch_table). - - For more details on function usage: - SELECT {schema_madlib}.{method}('usage') - """.format(**locals()) - - usage = """ - --------------------------------------------------------------------------- - USAGE - --------------------------------------------------------------------------- - SELECT {schema_madlib}.{method}( - keras_model_arch_table VARCHAR, -- Table containing Keras model architectures. - model_id INTEGER -- The id of the model arch to be deleted. - ); - - - --------------------------------------------------------------------------- - OUTPUT - --------------------------------------------------------------------------- - This method deletes the row corresponding to the given model_id in - keras_model_arch_table. This also tries to drop the table if the table is - empty after dropping the model_id. If there are any views depending on the - table, a warning message is displayed and the table is not dropped. - - --------------------------------------------------------------------------- - """.format(**locals()) - - return KerasModelArchDocumentation._returnHelpMsg( - schema_madlib, message, summary, usage, method) diff --git a/src/ports/postgres/modules/deep_learning/keras_model_arch_table.sql_in b/src/ports/postgres/modules/deep_learning/keras_model_arch_table.sql_in deleted file mode 100644 index cc915bb..0000000 --- a/src/ports/postgres/modules/deep_learning/keras_model_arch_table.sql_in +++ /dev/null @@ -1,429 +0,0 @@ -/* ----------------------------------------------------------------------- *//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * - * @file model_arch_table.sql_in - * - * @brief SQL functions for multilayer perceptron - * @date June 2012 - * - * - *//* ----------------------------------------------------------------------- */ - -m4_include(`SQLCommon.m4') -/** -@addtogroup grp_keras_model_arch - -@brief Utility function to load model architectures and weights into a table. - -\warning This MADlib method is still in early stage development. -Interface and implementation are subject to change. - - - -This utility function loads model architectures and -weights into a table for use by deep learning algorithms. -Model architecture is in JSON form -and model weights are in the form of PostgreSQL binary data types (bytea). -If the output table already exists, a new row is inserted -into the table so it can act as a repository for multiple model -architectures and weights. - -There is also a utility function to delete a model -from the table. - -@anchor load_keras_model -@par Load Model - -
-load_keras_model(
-    keras_model_arch_table,
-    model_arch,
-    model_weights,
-    name,
-    description
-    )
-
-\b Arguments -
-
keras_model_arch_table
-
VARCHAR. Output table to load keras model architecture and weights. -
- -
model_arch
-
JSON. JSON of the model architecture to load. -
- -
model_weights (optional)
-
bytea. Model weights to load as a PostgreSQL - binary data type. -
- -
name (optional)
-
TEXT, default: NULL. Free text string to provide - a name, if desired. -
- -
description (optional)
-
TEXT, default: NULL. Free text string to provide - a description, if desired. -
- -
- -Output table -
- The output table contains the following columns: - - - - - - - - - - - - - - - - - - - - - - - - - -
model_idSERIAL PRIMARY KEY. Model ID. -
model_archJSON. JSON blob of the model architecture. -
model_weightsBYTEA. Weights of the model which may be used for warm start - or transfer learning. - Weights are stored as a PostgreSQL binary data type. -
nameTEXT. Name of model (free text). -
descriptionTEXT. Description of model (free text). -
__internal_madlib_id__TEXT. Unique id for model arch. This is an id used internally be MADlib. -
-
- -@anchor delete_keras_model -@par Delete Model - -
-delete_keras_model(
-    keras_model_arch_table
-    model_id
-)
-
-\b Arguments -
-
keras_model_arch_table
-
VARCHAR. Table containing model architectures and weights. -
- -
model_id
-
INTEGER. The id of the model to be deleted. -
-
- -@anchor example -@par Examples --# Define model architecture. Use Keras to define -the model architecture: -
-import keras
-from keras.models import Sequential
-from keras.layers import Dense
-model_simple = Sequential()
-model_simple.add(Dense(10, activation='relu', input_shape=(4,)))
-model_simple.add(Dense(10, activation='relu'))
-model_simple.add(Dense(3, activation='softmax'))
-model_simple.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type)                 Output Shape              Param #
-=================================================================
-dense_1 (Dense)              (None, 10)                50
-_________________________________________________________________
-dense_2 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_3 (Dense)              (None, 3)                 33
-=================================================================
-Total params: 193
-Trainable params: 193
-Non-trainable params: 0
-\endverbatim
-
-Export the model to JSON: -
-model_simple.to_json()
-
-
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-
--# Load into model architecture table: -
-DROP TABLE IF EXISTS model_arch_library;
-SELECT madlib.load_keras_model('model_arch_library',  -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json,  -- JSON blob
-                               NULL,                  -- Weights
-                               'Sophie',              -- Name
-                               'A simple model'       -- Descr
-);
-SELECT COUNT(*) FROM model_arch_library;
-
-
- count
--------+
-     1
-
-Load another model architecture: -
-SELECT madlib.load_keras_model('model_arch_library',  -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json,  -- JSON blob
-                               NULL,                  -- Weights
-                               'Maria',               -- Name
-                               'Also a simple model'  -- Descr
-);
-SELECT COUNT(*) FROM model_arch_library;
-
-
- count
--------+
-     2
-
--# Load model weights. To load weights from previous -MADlib run, use UPDATE to load directly into the table. -For example, if 'model_weights' are the weights in the -output table 'iris_model' from a previous run -of 'madlib_keras_fit()' : -
-UPDATE model_arch_library SET model_weights = model_weights FROM iris_model WHERE model_id = 2;
-
-To load weights from Keras using a PL/Python function, -we need to flatten then serialize the weights to store -as a PostgreSQL binary data type. Byte format is more -efficient on space and memory compared to a numeric array. -The model weights will be de-serialized when -passed to Keras functions. -
-CREATE OR REPLACE FUNCTION load_weights() RETURNS VOID AS
-$$
-from keras.layers import *
-from keras import Sequential
-import numpy as np
-import plpy
-\#
-\# create model
-model = Sequential()
-model.add(Dense(10, activation='relu', input_shape=(4,)))
-model.add(Dense(10, activation='relu'))
-model.add(Dense(3, activation='softmax'))
-\#
-\# get weights, flatten and serialize
-weights = model.get_weights()
-weights_flat = [w.flatten() for w in weights]
-weights1d =  np.concatenate(weights_flat).ravel()
-weights_bytea = weights1d.tostring()
-\#
-\# load query
-load_query = plpy.prepare("""SELECT madlib.load_keras_model(
-                        'model_arch_library',
-                        $1, $2)
-                    """, ['json','bytea'])
-plpy.execute(load_query, [model.to_json(), weights_bytea])
-$$ language plpythonu;
--- Call load function
-SELECT load_weights();
--- Check weights loaded OK
-SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;
-
-
- count
--------+
-     1
-
-Load weights from Keras using psycopg2. (Psycopg is a PostgreSQL database adapter for the -Python programming language.) As above we need to flatten then serialize the weights to store as a -PostgreSQL binary data type. Note that the psycopg2.Binary function used below will increase the size of the -Python object for the weights, so if your model is large it might be better to use a PL/Python function as above. -
-import psycopg2
-import psycopg2 as p2
-conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')
-cur = conn.cursor()
-from keras.layers import *
-from keras import Sequential
-import numpy as np
-\#
-\# create model
-model = Sequential()
-model.add(Dense(10, activation='relu', input_shape=(4,)))
-model.add(Dense(10, activation='relu'))
-model.add(Dense(3, activation='softmax'))
-\#
-\# get weights, flatten and serialize
-weights = model.get_weights()
-weights_flat = [w.flatten() for w in weights]
-weights1d =  np.concatenate(weights_flat).ravel()
-weights_bytea = psycopg2.Binary(weights1d.tostring())
-\#
-\# load query
-query = "SELECT madlib.load_keras_model('model_arch_library', %s,%s)"
-cur.execute(query,[model.to_json(),weights_bytea])
-conn.commit()
-
-From SQL check if weights loaded OK: -
-SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;
-
-
- count
--------+
-     2
-
--# Delete one of the models: -
-SELECT madlib.delete_keras_model('model_arch_library',   -- Output table
-                                  1                      -- Model id
-                                );
-SELECT COUNT(*) FROM model_arch_library;
-
-
- count
--------+
-     3
-
- -@anchor related -@par Related Topics - -See keras_model_arch_table.sql_in - -*/ - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model( - keras_model_arch_table VARCHAR, - model_arch JSON, - model_weights bytea, - name TEXT, - description TEXT -) - RETURNS VOID AS $$ - PythonFunctionBodyOnlyNoSchema(`deep_learning', `keras_model_arch_table') - from utilities.control import AOControl - with AOControl(False): - keras_model_arch_table.load_keras_model(**globals()) -$$ LANGUAGE plpythonu VOLATILE; - --- Function to add a keras model to arch table -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model( - keras_model_arch_table VARCHAR, - model_arch JSON -) -RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.load_keras_model($1, $2, NULL, NULL, NULL) -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model( - keras_model_arch_table VARCHAR, - model_arch JSON, - model_weights bytea -) - RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.load_keras_model($1, $2, $3, NULL, NULL) -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model( - keras_model_arch_table VARCHAR, - model_arch JSON, - model_weights bytea, - name TEXT -) - RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.load_keras_model($1, $2, $3, $4, NULL) -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - - - - - - --- Functions for online help -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model( - message VARCHAR -) RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(deep_learning, keras_model_arch_table) - return keras_model_arch_table.KerasModelArchDocumentation.load_keras_model_help(schema_madlib, message) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_keras_model() -RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(deep_learning, keras_model_arch_table) - return keras_model_arch_table.KerasModelArchDocumentation.load_keras_model_help(schema_madlib, '') -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - --- Function to delete a keras model from arch table -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.delete_keras_model( - keras_model_arch_table VARCHAR, - model_id INTEGER -) -RETURNS VOID AS $$ - PythonFunctionBodyOnlyNoSchema(`deep_learning',`keras_model_arch_table') - from utilities.control import AOControl - with AOControl(False): - keras_model_arch_table.delete_keras_model(**globals()) -$$ LANGUAGE plpythonu VOLATILE; - --- Functions for online help -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.delete_keras_model( - message VARCHAR -) RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(deep_learning, keras_model_arch_table) - return keras_model_arch_table.KerasModelArchDocumentation.delete_keras_model_help(schema_madlib, message) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.delete_keras_model() -RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(deep_learning, keras_model_arch_table) - return keras_model_arch_table.KerasModelArchDocumentation.delete_keras_model_help(schema_madlib, '') -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras.py_in deleted file mode 100644 index ee27554..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras.py_in +++ /dev/null @@ -1,944 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import datetime -import os -import plpy -import sys -import time - -from keras import backend as K -from keras.layers import * -from keras.models import * -from keras.optimizers import * -from keras.regularizers import * -from madlib_keras_helper import * -from madlib_keras_validator import * -from madlib_keras_wrapper import * -from model_arch_info import * - -from utilities.utilities import _assert -from utilities.utilities import add_postfix -from utilities.utilities import is_platform_pg -from utilities.utilities import get_segments_per_host -from utilities.utilities import get_seg_number -from utilities.utilities import madlib_version -from utilities.utilities import unique_string -from utilities.validate_args import get_expr_type -from utilities.control import MinWarning -import tensorflow as tf - -class SD_STORE: - SESS = 'sess' - SEGMENT_MODEL = 'segment_model' - - @staticmethod - def init_SD(SD, sess, segment_model): - SD[SD_STORE.SESS] = sess - SD[SD_STORE.SEGMENT_MODEL] = segment_model - - @staticmethod - def clear_SD(SD): - del SD[SD_STORE.SEGMENT_MODEL] - del SD[SD_STORE.SESS] - -def get_init_model_and_sess(SD, device_name, gpu_count, segments_per_host, - model_architecture, compile_params): - # If a live session is present, re-use it. Otherwise, recreate it. - if SD_STORE.SESS in SD : - if SD_STORE.SEGMENT_MODEL not in SD: - plpy.error("Session and model should exist in SD after the first row" - "of the first iteration") - sess = SD[SD_STORE.SESS] - segment_model = SD[SD_STORE.SEGMENT_MODEL] - K.set_session(sess) - else: - sess = get_keras_session(device_name, gpu_count, segments_per_host) - K.set_session(sess) - segment_model = init_model(model_architecture, compile_params) - SD_STORE.init_SD(SD, sess, segment_model) - return segment_model, sess - -@MinWarning("warning") -def fit(schema_madlib, source_table, model, model_arch_table, - model_id, compile_params, fit_params, num_iterations, - use_gpus, validation_table=None, - metrics_compute_frequency=None, warm_start=False, name="", - description="", **kwargs): - - module_name = 'madlib_keras_fit' - fit_params = "" if not fit_params else fit_params - _assert(compile_params, "Compile parameters cannot be empty or NULL.") - - mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL - mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL - - dep_shape_col = add_postfix( - MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL, "_shape") - ind_shape_col = add_postfix( - MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, "_shape") - - segments_per_host = get_segments_per_host() - use_gpus = use_gpus if use_gpus else False - if use_gpus: - accessible_gpus_for_seg = get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name) - else: - accessible_gpus_for_seg = get_seg_number()*[0] - - fit_validator = FitInputValidator( - source_table, validation_table, model, model_arch_table, - model_id, mb_dep_var_col, mb_indep_var_col, - num_iterations, metrics_compute_frequency, warm_start, - use_gpus, accessible_gpus_for_seg) - if metrics_compute_frequency is None: - metrics_compute_frequency = num_iterations - - - warm_start = bool(warm_start) - - # The following two times must be recorded together. - metrics_elapsed_start_time = time.time() - start_training_time = datetime.datetime.now() - #TODO add a unit test for this in a future PR - # save the original value of the env variable so that we can reset it later. - original_cuda_env = None - if CUDA_VISIBLE_DEVICES_KEY in os.environ: - original_cuda_env = os.environ[CUDA_VISIBLE_DEVICES_KEY] - - # Get the serialized master model - start_deserialization = time.time() - model_arch, model_weights = get_model_arch_weights(model_arch_table, model_id) - num_classes = get_num_classes(model_arch) - input_shape = get_input_shape(model_arch) - fit_validator.validate_input_shapes(input_shape) - dist_key_col = '0' if is_platform_pg() else DISTRIBUTION_KEY_COLNAME - gp_segment_id_col = '0' if is_platform_pg() else GP_SEGMENT_ID_COLNAME - - serialized_weights = get_initial_weights(model, model_arch, model_weights, - warm_start, use_gpus, accessible_gpus_for_seg) - # Compute total images on each segment - dist_key_mapping, images_per_seg_train = get_image_count_per_seg_for_minibatched_data_from_db(source_table) - - - if validation_table: - seg_ids_val, images_per_seg_val = get_image_count_per_seg_for_minibatched_data_from_db(validation_table) - - # Construct validation dataset if provided - validation_set_provided = bool(validation_table) - validation_metrics = []; validation_loss = [] - - # Prepare the SQL for running distributed training via UDA - compile_params_to_pass = "$madlib$" + compile_params + "$madlib$" - fit_params_to_pass = "$madlib$" + fit_params + "$madlib$" - run_training_iteration = plpy.prepare(""" - SELECT {schema_madlib}.fit_step( - {mb_dep_var_col}, - {mb_indep_var_col}, - {dep_shape_col}, - {ind_shape_col}, - $MAD${model_arch}$MAD$::TEXT, - {compile_params_to_pass}::TEXT, - {fit_params_to_pass}::TEXT, - {dist_key_col}, - ARRAY{dist_key_mapping}, - {gp_segment_id_col}, - {segments_per_host}, - ARRAY{images_per_seg_train}, - {use_gpus}::BOOLEAN, - ARRAY{accessible_gpus_for_seg}, - $1, - $2 - ) AS iteration_result - FROM {source_table} - """.format(**locals()), ["bytea", "boolean"]) - - # Define the state for the model and loss/metric storage lists - training_loss, training_metrics, metrics_elapsed_time = [], [], [] - metrics_iters = [] - - # get the size of serialized model weights string in KB - model_size = sys.getsizeof(serialized_weights)/1024.0 - - # Run distributed training for specified number of iterations - for i in range(1, num_iterations+1): - start_iteration = time.time() - is_final_iteration = (i == num_iterations) - serialized_weights = plpy.execute(run_training_iteration, - [serialized_weights, is_final_iteration] - )[0]['iteration_result'] - end_iteration = time.time() - info_str = "\tTime for training in iteration {0}: {1} sec".format(i, - end_iteration - start_iteration) - - if should_compute_metrics_this_iter(i, metrics_compute_frequency, - num_iterations): - # Compute loss/accuracy for training data. - compute_out = compute_loss_and_metrics( - schema_madlib, source_table, compile_params_to_pass, model_arch, - serialized_weights, use_gpus, accessible_gpus_for_seg, dist_key_mapping, - images_per_seg_train, training_metrics, training_loss, i, is_final_iteration) - metrics_iters.append(i) - compute_time, compute_metrics, compute_loss = compute_out - - info_str += "\n\tTime for evaluating training dataset in "\ - "iteration {0}: {1} sec\n".format(i, compute_time) - info_str += "\tTraining set metric after iteration {0}: {1}\n".format( - i, compute_metrics) - info_str += "\tTraining set loss after iteration {0}: {1}".format( - i, compute_loss) - - if validation_set_provided: - # Compute loss/accuracy for validation data. - val_compute_out = compute_loss_and_metrics( - schema_madlib, validation_table, compile_params_to_pass, - model_arch, serialized_weights, use_gpus, accessible_gpus_for_seg, - seg_ids_val, images_per_seg_val, validation_metrics, - validation_loss, i, is_final_iteration) - val_compute_time, val_compute_metrics, val_compute_loss = val_compute_out - - info_str += "\n\tTime for evaluating validation dataset in "\ - "iteration {0}: {1} sec\n".format(i, val_compute_time) - info_str += "\tValidation set metric after iteration {0}: {1}\n".format( - i, val_compute_metrics) - info_str += "\tValidation set loss after iteration {0}: {1}".format( - i, val_compute_loss) - - metrics_elapsed_end_time = time.time() - metrics_elapsed_time.append( - metrics_elapsed_end_time-metrics_elapsed_start_time) - plpy.info("\n"+info_str) - end_training_time = datetime.datetime.now() - - version = madlib_version(schema_madlib) - src_summary_dict = get_source_summary_table_dict(fit_validator) - class_values = src_summary_dict['class_values'] - class_values_type = src_summary_dict['class_values_type'] - norm_const = src_summary_dict['norm_const'] - norm_const_type = src_summary_dict['norm_const_type'] - dep_vartype = src_summary_dict['dep_vartype'] - dependent_varname = src_summary_dict['dependent_varname_in_source_table'] - independent_varname = src_summary_dict['independent_varname_in_source_table'] - # Define some constants to be inserted into the summary table. - model_type = "madlib_keras" - compile_params_dict = convert_string_of_args_to_dict(compile_params) - metrics_list = get_metrics_from_compile_param(compile_params) - is_metrics_specified = True if metrics_list else False - metrics_type = 'ARRAY{0}'.format(metrics_list) if is_metrics_specified else 'NULL' - metrics_iters = metrics_iters if metrics_iters else 'NULL' - # We always compute the training loss and metrics, at least once. - training_loss_final = training_loss[-1] - training_loss = 'ARRAY{0}'.format(training_loss) if training_loss else 'NULL' - training_metrics_final, training_metrics = get_metrics_sql_string( - training_metrics, is_metrics_specified) - # Validation loss and metrics are computed only if validation_table - # is provided. - if validation_set_provided: - validation_metrics_final, validation_metrics = get_metrics_sql_string( - validation_metrics, is_metrics_specified) - validation_loss_final = validation_loss[-1] - validation_loss = 'ARRAY{0}'.format(validation_loss) - # Must quote the string before inserting to table. Explicitly - # quoting it here since this can also take a NULL value, done - # in the else part. - validation_table = "$MAD${0}$MAD$".format(validation_table) - else: - validation_metrics = validation_loss = 'NULL' - validation_metrics_final = validation_loss_final = 'NULL' - validation_table = 'NULL' - - if warm_start: - plpy.execute("DROP TABLE {0}, {1}".format - (model, fit_validator.output_summary_model_table)) - create_output_summary_table = plpy.prepare(""" - CREATE TABLE {output_summary_model_table} AS - SELECT - $MAD${source_table}$MAD$::TEXT AS source_table, - $MAD${model}$MAD$::TEXT AS model, - $MAD${dependent_varname}$MAD$::TEXT AS dependent_varname, - $MAD${independent_varname}$MAD$::TEXT AS independent_varname, - $MAD${model_arch_table}$MAD$::TEXT AS model_arch_table, - {model_id}::INTEGER AS {model_id_colname}, - $1 AS compile_params, - $2 AS fit_params, - {num_iterations}::INTEGER AS num_iterations, - {validation_table}::TEXT AS validation_table, - {metrics_compute_frequency}::INTEGER AS metrics_compute_frequency, - $3 AS name, - $4 AS description, - '{model_type}'::TEXT AS model_type, - {model_size}::DOUBLE PRECISION AS model_size, - '{start_training_time}'::TIMESTAMP AS start_training_time, - '{end_training_time}'::TIMESTAMP AS end_training_time, - $5 AS metrics_elapsed_time, - '{version}'::TEXT AS madlib_version, - {num_classes}::INTEGER AS num_classes, - $6 AS {class_values_colname}, - $MAD${dep_vartype}$MAD$::TEXT AS {dependent_vartype_colname}, - {norm_const}::{FLOAT32_SQL_TYPE} AS {normalizing_const_colname}, - {metrics_type}::TEXT[] AS metrics_type, - {training_metrics_final}::DOUBLE PRECISION AS training_metrics_final, - {training_loss_final}::DOUBLE PRECISION AS training_loss_final, - {training_metrics}::DOUBLE PRECISION[] AS training_metrics, - {training_loss}::DOUBLE PRECISION[] AS training_loss, - {validation_metrics_final}::DOUBLE PRECISION AS validation_metrics_final, - {validation_loss_final}::DOUBLE PRECISION AS validation_loss_final, - {validation_metrics}::DOUBLE PRECISION[] AS validation_metrics, - {validation_loss}::DOUBLE PRECISION[] AS validation_loss, - ARRAY{metrics_iters}::INTEGER[] AS metrics_iters - """.format(output_summary_model_table=fit_validator.output_summary_model_table, - class_values_colname=CLASS_VALUES_COLNAME, - dependent_vartype_colname=DEPENDENT_VARTYPE_COLNAME, - normalizing_const_colname=NORMALIZING_CONST_COLNAME, - FLOAT32_SQL_TYPE = FLOAT32_SQL_TYPE, - model_id_colname = ModelArchSchema.MODEL_ID, - **locals()), - ["TEXT", "TEXT", "TEXT", "TEXT", "DOUBLE PRECISION[]", class_values_type]) - plpy.execute(create_output_summary_table, - [compile_params, fit_params, name, - description, metrics_elapsed_time, class_values]) - - plpy.execute(""" - CREATE TABLE {0} - (model_weights bytea, - {1} json)""".format(model, ModelArchSchema.MODEL_ARCH)) - insert_output_table = plpy.prepare(""" - INSERT INTO {0} SELECT model_weights, {1} - FROM (VALUES($1, $2))t(model_weights, {1}) - """.format(model, ModelArchSchema.MODEL_ARCH), ["bytea", "json"]) - plpy.execute(insert_output_table, [serialized_weights, model_arch]) - - #TODO add a unit test for this in a future PR - reset_cuda_env(original_cuda_env) - -def get_initial_weights(model_table, model_arch, serialized_weights, warm_start, - use_gpus, accessible_gpus_for_seg, mst_filter=''): - """ - If warm_start is True, return back initial weights from model table. - If warm_start is False, first try to get the weights from model_arch - table, if no weights are defined there, randomly initialize it using - keras. - We also need to set the cuda environment variable based on the platform. - 1. For postgres, if user specifies use_gpus=False which means they want - to use CPU, then we have to set CUDA_VISIBLE_DEVICES to -1 to disable gpu. - Otherwise model.get_weights() will use gpu if available. - - 2. For gpdb, we want to disable gpu on gpdb's master node because GPUs - will only be used for segment nodes. - @args: - @param model_table: Output model table passed in to fit. - @param model_arch_result: Dict containing model architecture info. - @param warm_start: Boolean flag indicating warm start or not. - """ - if is_platform_pg(): - _ = get_device_name_and_set_cuda_env(accessible_gpus_for_seg[0], None) - else: - _ = get_device_name_and_set_cuda_env(0, None) - - if warm_start: - serialized_weights = plpy.execute(""" - SELECT model_weights FROM {model_table} {mst_filter} LIMIT 1 - """.format(**locals()))[0]['model_weights'] - else: - if not serialized_weights: - model = model_from_json(model_arch) - serialized_weights = madlib_keras_serializer.serialize_nd_weights( - model.get_weights()) - return serialized_weights - -def get_source_summary_table_dict(fit_validator): - source_summary = plpy.execute(""" - SELECT - {class_values} AS class_values, - {norm_const} AS norm_const, - {dep_vartype} AS dep_vartype, - {dep_varname} AS dependent_varname_in_source_table, - {indep_varname} AS independent_varname_in_source_table - FROM {tbl} - """.format(class_values=CLASS_VALUES_COLNAME, - norm_const=NORMALIZING_CONST_COLNAME, - dep_vartype=DEPENDENT_VARTYPE_COLNAME, - dep_varname='dependent_varname', - indep_varname='independent_varname', - tbl=fit_validator.source_summary_table))[0] - source_summary['class_values_type'] = get_expr_type( - CLASS_VALUES_COLNAME, fit_validator.source_summary_table) - source_summary['norm_const_type'] = get_expr_type( - NORMALIZING_CONST_COLNAME, fit_validator.source_summary_table) - return source_summary - -def get_metrics_sql_string(metrics_list, is_metrics_specified): - """ - Return the SQL string to use for creating metrics SQL values. - """ - if is_metrics_specified: - metrics_final = metrics_list[-1] - metrics_all = 'ARRAY{0}'.format(metrics_list) - else: - metrics_final = metrics_all = 'NULL' - return metrics_final, metrics_all - -def compute_loss_and_metrics(schema_madlib, table, compile_params, model_arch, - serialized_weights, use_gpus, accessible_gpus_for_seg, - dist_key_mapping, images_per_seg_val, metrics_list, loss_list, - curr_iter, is_final_iteration): - """ - Compute the loss and metric using a given model (serialized_weights) on the - given dataset (table.) - """ - start_val = time.time() - evaluate_result = get_loss_metric_from_keras_eval(schema_madlib, - table, - compile_params, - model_arch, - serialized_weights, - use_gpus, - accessible_gpus_for_seg, - dist_key_mapping, - images_per_seg_val, - is_final_iteration) - end_val = time.time() - - if len(evaluate_result) not in [1, 2]: - plpy.error('Calling evaluate on table {0} returned < 2 ' - 'metrics. Expected both loss and a metric.'.format(table)) - loss = evaluate_result[0] - metric = evaluate_result[1] - metrics_list.append(metric) - loss_list.append(loss) - return end_val - start_val, metric, loss - -def should_compute_metrics_this_iter(curr_iter, metrics_compute_frequency, - num_iterations): - """ - Check if we want to compute loss/accuracy for the current iteration - :param curr_iter: - :param metrics_compute_frequency: - :param num_iterations: - :return: Returns a boolean - return TRUE, if it is the last iteration, or if metrics_compute_frequency - iterations have elapsed since the last time it was computed. - return FALSE otherwise. - """ - # Compute loss/accuracy every metrics_compute_frequency'th iteration, - # and also for the last iteration. - return (curr_iter)%metrics_compute_frequency == 0 or \ - curr_iter == num_iterations - -def init_model(model_architecture, compile_params): - """ - Should only be called at the first row of first iteration. - """ - segment_model = model_from_json(model_architecture) - compile_model(segment_model, compile_params) - return segment_model - -def update_model(segment_model, prev_serialized_weights): - """ - Happens at first row of each iteration. - """ - model_shapes = get_model_shapes(segment_model) - model_weights = madlib_keras_serializer.deserialize_as_nd_weights( - prev_serialized_weights, model_shapes) - segment_model.set_weights(model_weights) - -def fit_transition(state, dependent_var, independent_var, dependent_var_shape, - independent_var_shape, model_architecture, - compile_params, fit_params, dist_key, dist_key_mapping, - current_seg_id, segments_per_host, images_per_seg, use_gpus, - accessible_gpus_for_seg, prev_serialized_weights, is_final_iteration=True, - is_multiple_model=False, **kwargs): - """ - This transition function is common for madlib_keras_fit() and - madlib_keras_fit_multiple_model(). The important difference between - these two calls is the way this function handles the input param - prev_serialized_weights and clearing keras session. - For madlib_keras_fit_multiple_model, - a. prev_serialized_weights is always passed in as the state - (image count, serialized weights), since it is fetched in the - table for each hop of the model between segments. - b. keras session is cleared at the end of each iteration, i.e, - last row of each iteration. - For madlib_keras_fit, - a. prev_serialized_weights is passed in as serialized weights - b. keras session is cleared at the end of the final iteration, - i.e, last row of last iteration. - """ - if not independent_var or not dependent_var: - return state - SD = kwargs['SD'] - device_name = get_device_name_and_set_cuda_env(accessible_gpus_for_seg[current_seg_id], current_seg_id) - - segment_model, sess = get_init_model_and_sess(SD, device_name, - accessible_gpus_for_seg[current_seg_id], - segments_per_host, - model_architecture, compile_params) - if not state: - agg_image_count = 0 - set_model_weights(segment_model, prev_serialized_weights) - else: - agg_image_count = float(state) - - # Prepare the data - x_train = np_array_float32(independent_var, independent_var_shape) - y_train = np_array_int16(dependent_var, dependent_var_shape) - - # Fit segment model on data - #TODO consider not doing this every time - fit_params = parse_and_validate_fit_params(fit_params) - segment_model.fit(x_train, y_train, **fit_params) - - # Aggregating number of images, loss and accuracy - agg_image_count += len(x_train) - total_images = get_image_count_per_seg_from_array(dist_key_mapping.index(dist_key), - images_per_seg) - is_last_row = agg_image_count == total_images - return_state = get_state_to_return(segment_model, is_last_row, is_multiple_model, - agg_image_count, total_images) - if is_last_row: - if is_final_iteration or is_multiple_model: - SD_STORE.clear_SD(SD) - clear_keras_session(sess) - - return return_state - -def get_state_to_return(segment_model, is_last_row, is_multiple_model, agg_image_count, - total_images): - """ - 1. For both model averaging fit_transition and fit multiple transition, the - state only needs to have the image count except for the last row. - 2. For model averaging fit_transition, the last row state must always contain - the image count as well as the model weights - 3. For fit multiple transition, the last row state only needs the model - weights. This state is the output of the UDA for that hop. We don't need - the image_count here because unlike model averaging, model hopper does - not have a merge/final function and there is no need to average the weights - based on the image count. - :param segment_model: cached model for that segment - :param is_last_row: boolean to indicate if last row for that hop - :param is_multiple_model: boolean - :param agg_image_count: aggregated image count per hop - :param total_images: total images per segment - :return: - """ - if is_last_row: - updated_model_weights = segment_model.get_weights() - if is_multiple_model: - new_state = madlib_keras_serializer.serialize_nd_weights(updated_model_weights) - else: - updated_model_weights = [total_images * w for w in updated_model_weights] - new_state = madlib_keras_serializer.serialize_state_with_nd_weights( - agg_image_count, updated_model_weights) - else: - new_state = float(agg_image_count) - - return new_state - -def fit_merge(state1, state2, **kwargs): - - # Return if called early - if not state1 or not state2: - return state1 or state2 - - # Deserialize states - image_count1, weights1 = madlib_keras_serializer.deserialize_as_image_1d_weights(state1) - image_count2, weights2 = madlib_keras_serializer.deserialize_as_image_1d_weights(state2) - - # Compute total image counts - image_count = (image_count1 + image_count2) * 1.0 - - # Aggregate the weights - total_weights = weights1 + weights2 - - # Return the merged state - return madlib_keras_serializer.serialize_state_with_1d_weights( - image_count, total_weights) - -def fit_final(state, **kwargs): - # Return if called early - if not state: - return state - - image_count, weights = madlib_keras_serializer.deserialize_as_image_1d_weights(state) - if image_count == 0: - plpy.error("fit_final: Total images processed is 0") - - # Averaging the weights - weights /= image_count - return madlib_keras_serializer.serialize_nd_weights(weights) - - -def evaluate(schema_madlib, model_table, test_table, output_table, - use_gpus, mst_key, **kwargs): - - module_name = 'madlib_keras_evaluate' - is_mult_model = mst_key is not None - if test_table: - test_summary_table = add_postfix(test_table, "_summary") - model_summary_table = None - if model_table: - model_summary_table = add_postfix(model_table, "_summary") - - mult_where_clause = "" - if is_mult_model: - mult_where_clause = "WHERE mst_key = {0}".format(mst_key) - model_summary_table = create_summary_view(module_name, model_table, mst_key) - - validate_evaluate(module_name, model_table, model_summary_table, test_table, test_summary_table, output_table, is_mult_model) - - segments_per_host = get_segments_per_host() - if use_gpus: - accessible_gpus_for_seg = get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name) - else: - accessible_gpus_for_seg = get_seg_number()*[0] - - model_weights_query = "SELECT model_weights, model_arch FROM {0} {1}".format( - model_table, mult_where_clause) - - res = plpy.execute(model_weights_query)[0] - _assert(res, "{0}: The model does not exist.") - model_weights = res['model_weights'] - model_arch = res['model_arch'] - - input_shape = get_input_shape(model_arch) - InputValidator.validate_input_shape( - test_table, MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, input_shape, 2, True) - - compile_params_query = "SELECT compile_params, metrics_type FROM {0}".format(model_summary_table) - res = plpy.execute(compile_params_query)[0] - metrics_type = res['metrics_type'] - compile_params = "$madlib$" + res['compile_params'] + "$madlib$" - - dist_key_mapping, images_per_seg = get_image_count_per_seg_for_minibatched_data_from_db(test_table) - - loss, metric = \ - get_loss_metric_from_keras_eval( - schema_madlib, test_table, compile_params, model_arch, - model_weights, use_gpus, accessible_gpus_for_seg, dist_key_mapping, - images_per_seg) - - if not metrics_type: - metrics_type = None - metric = None - - with MinWarning("error"): - create_output_table = plpy.prepare(""" - CREATE TABLE {0} AS - SELECT $1 as loss, $2 as metric, $3 as metrics_type""".format(output_table), ["FLOAT", "FLOAT", "TEXT[]"]) - plpy.execute(create_output_table, [loss, metric, metrics_type]) - - if is_mult_model: - plpy.execute("DROP VIEW IF EXISTS {0}".format(model_summary_table)) - -def validate_evaluate(module_name, model_table, model_summary_table, test_table, test_summary_table, output_table, is_mult_model): - def _validate_test_summary_tbl(): - input_tbl_valid(test_summary_table, module_name, - error_suffix_str="Please ensure that the test table ({0}) " - "has been preprocessed by " - "the image preprocessor.".format(test_table)) - cols_in_tbl_valid(test_summary_table, [CLASS_VALUES_COLNAME, - NORMALIZING_CONST_COLNAME, DEPENDENT_VARTYPE_COLNAME, - DEPENDENT_VARNAME_COLNAME, INDEPENDENT_VARNAME_COLNAME], module_name) - - input_tbl_valid(model_table, module_name) - if is_mult_model and not columns_exist_in_table(model_table, ['mst_key']): - plpy.error("{module_name}: Single model should not pass mst_key".format(**locals())) - if not is_mult_model and columns_exist_in_table(model_table, ['mst_key']): - plpy.error("{module_name}: Multi-model needs to pass mst_key".format(**locals())) - InputValidator.validate_predict_evaluate_tables( - module_name, model_table, model_summary_table, - test_table, output_table, MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL) - _validate_test_summary_tbl() - validate_bytea_var_for_minibatch(test_table, - MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL) - -def get_loss_metric_from_keras_eval(schema_madlib, table, compile_params, - model_arch, serialized_weights, use_gpus, - accessible_gpus_for_seg, dist_key_mapping, images_per_seg, - is_final_iteration=True): - - dist_key_col = '0' if is_platform_pg() else DISTRIBUTION_KEY_COLNAME - gp_segment_id_col = '0' if is_platform_pg() else GP_SEGMENT_ID_COLNAME - segments_per_host = get_segments_per_host() - - mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL - mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL - - dep_shape_col = add_postfix( - MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL, "_shape") - ind_shape_col = add_postfix( - MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL, "_shape") - """ - This function will call the internal keras evaluate function to get the loss - and accuracy of each tuple which then gets averaged to get the final result. - """ - use_gpus = use_gpus if use_gpus else False - evaluate_query = plpy.prepare(""" - select ({schema_madlib}.internal_keras_evaluate( - {mb_dep_var_col}, - {mb_indep_var_col}, - {dep_shape_col}, - {ind_shape_col}, - $MAD${model_arch}$MAD$, - $1, - {compile_params}, - {dist_key_col}, - ARRAY{dist_key_mapping}, - {gp_segment_id_col}, - {segments_per_host}, - ARRAY{images_per_seg}, - {use_gpus}::BOOLEAN, - ARRAY{accessible_gpus_for_seg}, - {is_final_iteration} - )) as loss_metric - from {table} - """.format(**locals()), ["bytea"]) - res = plpy.execute(evaluate_query, [serialized_weights]) - loss_metric = res[0]['loss_metric'] - return loss_metric - - -def internal_keras_eval_transition(state, dependent_var, independent_var, - dependent_var_shape, independent_var_shape, - model_architecture, serialized_weights, compile_params, - dist_key, dist_key_mapping, current_seg_id, - segments_per_host, images_per_seg, - use_gpus, accessible_gpus_for_seg, - is_final_iteration, **kwargs): - SD = kwargs['SD'] - device_name = get_device_name_and_set_cuda_env(accessible_gpus_for_seg[current_seg_id], current_seg_id) - agg_loss, agg_metric, agg_image_count = state - - # This transition function is common to evaluate as well as the fit functions - # and is used to determine when to clear the session. - # For evaluate, - # is_final_iteration is always set to true, so the session is cleared once - # evaluated the last buffer on each segment. - # When called from fit functions, - # if is_final_iteration is false, the fit function has already created a - # session and a graph that can be used between iterations and cleared only - # for the last buffer of last iteration - # if is_final_iteration is false, we can clear the - - segment_model, sess = get_init_model_and_sess(SD, device_name, - accessible_gpus_for_seg[current_seg_id], - segments_per_host, - model_architecture, - compile_params) - if not agg_image_count: - # These should already be 0, but just in case make sure - agg_metric = 0 - agg_loss = 0 - set_model_weights(segment_model, serialized_weights) - - x_val = np_array_float32(independent_var, independent_var_shape) - y_val = np_array_int16(dependent_var, dependent_var_shape) - - with K.tf.device(device_name): - res = segment_model.evaluate(x_val, y_val) - - # if metric is None, model.evaluate will only return loss as a scalar - # Otherwise, it will return a list which has loss and metric - if type(res) is list: - loss, metric = res - else: - loss = res - metric = 0 - - image_count = len(y_val) - - agg_image_count += image_count - agg_loss += (image_count * loss) - agg_metric += (image_count * metric) - - total_images = get_image_count_per_seg_from_array(dist_key_mapping.index(dist_key), - images_per_seg) - - if agg_image_count == total_images and is_final_iteration: - K.clear_session() - sess.close() - SD_STORE.clear_SD(SD) - del segment_model - del sess - - state[0] = agg_loss - state[1] = agg_metric - state[2] = agg_image_count - - return state - -def internal_keras_eval_merge(state1, state2, **kwargs): - # If either state is None, return the other one - if not state1 or not state2: - return state1 or state2 - - loss1, metric1, image_count1 = state1 - loss2, metric2, image_count2 = state2 - - merged_loss = loss1 + loss2 - merged_metric = metric1 + metric2 - total_image_count = image_count1 + image_count2 - - merged_state = [ merged_loss, merged_metric , total_image_count ] - - return merged_state - -def internal_keras_eval_final(state, **kwargs): - loss, metric, image_count = state - - if image_count == 0: - plpy.error("internal_keras_eval_final: Total images processed is 0") - - loss /= image_count - metric /= image_count - - return loss, metric - -def fit_help(schema_madlib, message, **kwargs): - """ - Help function for keras fit - - Args: - @param schema_madlib - @param message: string, Help message string - @param kwargs - - Returns: - String. Help/usage information - """ - if not message: - help_string = """ ------------------------------------------------------------------------ - SUMMARY ------------------------------------------------------------------------ -This module allows you to use SQL to call deep learning -models designed in Keras, which is a high-level neural -network API written in Python. -Keras was developed for fast experimentation. It can run -on top of different backends and the one that is currently -supported by MADlib is TensorFlow. The implementation -in MADlib is distributed and designed to train -a single large model across multiple segments (workers) -in a Greenplum database. PostgreSQL is also supported. - -For more details on function usage: - SELECT {schema_madlib}.madlib_keras_fit('usage') - """ - elif message in ['usage', 'help', '?']: - help_string = """ ------------------------------------------------------------------------ - USAGE ------------------------------------------------------------------------ - SELECT {schema_madlib}.madlib_keras_fit( - source_table, -- Name of the table containing the - training data - model, -- Name of the output table containing - the model - model_arch_table, -- Name of the table containing the - model architecture - model_id, -- This is the id in 'model_arch_table' - containing the model architecture - compile_params, -- Parameters passed to the compile - method of the Keras model class - fit_params, -- Parameters passed to the fit method - of the Keras model class - num_iterations, -- Number of iterations to train. - use_gpus, -- Flag to enable GPU support - validation_table, -- Name of the table containing - the validation dataset - metrics_compute_frequency, -- Frequency to compute per-iteration - metrics - warm_start, -- Flag to enable warm start - name, -- Free text string to identify a name - description -- Free text string to provide a description - ) - ); - ------------------------------------------------------------------------ - OUTPUT ------------------------------------------------------------------------ -The output table ('model' above) contains the following columns: - -model_weights: Byte array containing the weights of the neural net. -model_arch: A JSON representation of the model architecture used in - training. - -A summary table ('_summary') is created to store various training -statistics as well as the input parameters. -""" - else: - help_string = "No such option. Use {schema_madlib}.madlib_keras_fit()" - - return help_string.format(schema_madlib=schema_madlib) -# --------------------------------------------------------------------- - - -def evaluate_help(schema_madlib, message, **kwargs): - """ - Help function for keras evaluate - - Args: - @param schema_madlib - @param message: string, Help message string - @param kwargs - - Returns: - String. Help/usage information - """ - if not message: - help_string = """ ------------------------------------------------------------------------ - SUMMARY ------------------------------------------------------------------------ -This function allows the user to evaluate a madlib_keras_fit trained -model. - -For more details on function usage: - SELECT {schema_madlib}.madlib_keras_evaluate('usage') - """ - elif message in ['usage', 'help', '?']: - help_string = """ ------------------------------------------------------------------------ - USAGE ------------------------------------------------------------------------ - SELECT {schema_madlib}.madlib_keras_evaluate( - model_table, -- Name of the table containing the model - test_table, -- Name of the table containing the evaluation dataset - output_table, -- Name of the output table - use_gpus, -- Flag to enable GPU support - mst_key -- Identifier for the desired model out of multimodel - training output - ) - ); - ------------------------------------------------------------------------ - OUTPUT ------------------------------------------------------------------------ -The output table ('output_table' above) contains the following columns: - -loss: Loss value on evaluation dataset. -metric: Metric value on evaluation dataset, where 'metrics_type' - below identifies the type of metric. -metrics_type: Type of metric used that was used in the training step. -""" - else: - help_string = "No such option. Use {schema_madlib}.madlib_keras_evaluate()" - - return help_string.format(schema_madlib=schema_madlib) -# --------------------------------------------------------------------- - diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in deleted file mode 100644 index 90e7a98..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in +++ /dev/null @@ -1,2162 +0,0 @@ -/* ----------------------------------------------------------------------- *//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * - * @file madlib_keras.sql_in - * - * @brief SQL functions for distributed deep learning with keras - * @date June 2019 - * - * - *//* ----------------------------------------------------------------------- */ - -m4_include(`SQLCommon.m4') - -/** -@addtogroup grp_keras - -@brief Fit, evaluate and predict using the Keras API. - - - -\warning This MADlib method is still in early stage development. -Interface and implementation are subject to change. - -This module allows you to use SQL to call deep learning -models designed in Keras [1], which is a high-level neural -network API written in Python. -Keras was developed for fast experimentation. It can run -on top of different backends and the one that is currently -supported by MADlib is TensorFlow [2]. The implementation -in MADlib is distributed and designed to train -a single model across multiple segments (workers) -in Greenplum database. (PostgreSQL is also supported.) -Alternatively, to train multiple models at the same time for model -architecture search or hyperparameter tuning, you can -use Model Selection. - -The main use case is image classification -using sequential models, which are made up of a -linear stack of layers. This includes multilayer perceptrons (MLPs) -and convolutional neural networks (CNNs). Regression is not -currently supported. - -Before using Keras in MADlib you will need to mini-batch -your training and evaluation datasets by calling the -Preprocessor -for Images which is a utility that prepares image data for -use by models that support mini-batch as an optimization option. -This is a one-time operation and you would only -need to re-run the preprocessor if your input data has changed. -The advantage of using mini-batching is that it -can perform better than stochastic gradient descent -because it uses more than one training example at a time, -typically resulting faster and smoother convergence [3]. - -You can also do inference on models that have not been trained in MADlib, -but rather imported from an external source. This is in the section -called "Predict BYOM" below, where "BYOM" stands for "Bring Your Own Model." - -Note that the following MADlib functions are targeting a specific Keras -version (2.2.4) with a specific TensorFlow kernel version (1.14). -Using a newer or older version may or may not work as intended. - -@note CUDA GPU memory cannot be released until the process holding it is terminated. -When a MADlib deep learning function is called with GPUs, Greenplum internally -creates a process (called a slice) which calls TensorFlow to do the computation. -This process holds the GPU memory until one of the following two things happen: -query finishes and user logs out of the Postgres client/session; or, -query finishes and user waits for the timeout set by gp_vmem_idle_resource_timeout. -The default value for this timeout is 18 sec [8]. So the recommendation is: -log out/reconnect to the session after every GPU query; or -wait for gp_vmem_idle_resource_timeout before you run another GPU query (you can -also set it to a lower value). - -@anchor keras_fit -@par Fit -The fit (training) function has the following format: - -
-madlib_keras_fit(
-    source_table,
-    model,
-    model_arch_table,
-    model_id,
-    compile_params,
-    fit_params,
-    num_iterations,
-    use_gpus,
-    validation_table,
-    metrics_compute_frequency,
-    warm_start,
-    name,
-    description
-    )
-
- -\b Arguments -
-
source_table
-
TEXT. Name of the table containing the training data. - This is the name of the output - table from the image preprocessor. Independent - and dependent variables are specified in the preprocessor - step which is why you do not need to explictly state - them here as part of the fit function.
- -
model
-
TEXT. Name of the output table containing the model. - Details of the output table are shown below. -
- -
model_arch_table
-
TEXT. Name of the table containing the model - architecture and (optionally) initial weights to use for - training. -
- -
model_id
-
INTEGER. This is the id in 'model_arch_table' - containing the model architecture and (optionally) - initial weights to use for training. -
- -
compile_params
-
TEXT. - Parameters passed to the compile method of the Keras - model class [4]. These parameters will be passed through as is - so they must conform to the Keras API definition. - As an example, you might use something like: loss='categorical_crossentropy', optimizer='adam', metrics=['acc']. - The mandatory parameters that must be specified are 'optimizer' - and 'loss'. Others are optional and will use the default - values as per Keras if not specified here. Also, when - specifying 'loss' and 'metrics' do not include the - module and submodule prefixes - like loss='losses.categorical_crossentropy' - or optimizer='keras.optmizers.adam'. - - @note - The following loss function is - not supported: sparse_categorical_crossentropy. - The following metrics are not - supported: sparse_categorical_accuracy, top_k_categorical_accuracy, sparse_top_k_categorical_accuracy and custom metrics. - -
- -
fit_params
-
TEXT. Parameters passed to the fit method of the Keras - model class [4]. These will be passed through as is - so they must conform to the Keras API definition. - As an example, you might use something like: - batch_size=128, epochs=4. - There are no mandatory parameters so - if you specify NULL, it will use all default - values as per Keras. -
- -
num_iterations
-
INTEGER. Number of iterations to train. -
- -
use_gpus (optional)
-
BOOLEAN, default: FALSE (i.e., CPU). Determines whether GPUs - are to be used for training the neural network. Set to TRUE to use GPUs. - - @note - This parameter must not conflict with how the distribution rules are set in - the preprocessor function. For example, if you set a distribution rule to use - certain segments on hosts that do not have GPUs attached, you will get an error - if you set ‘use_gpus’ to TRUE. Also, we have seen some memory related issues - when segments share GPU resources. - For example, if you have 1 GPU per segment host and your cluster has 4 - segments per segment host, it means that all 4 - segments will share the same - GPU on each host. The current recommended - configuration is 1 GPU per segment. -
- -
validation_table (optional)
-
TEXT, default: none. Name of the table containing - the validation dataset. - Note that the validation dataset must be preprocessed - in the same way as the training dataset, so this - is the name of the output - table from running the image preprocessor on the validation dataset. - Using a validation dataset can mean a - longer training time, depending on its size. - This can be controlled using the 'metrics_compute_frequency' - paremeter described below.
- -
metrics_compute_frequency (optional)
-
INTEGER, default: once at the end of training - after 'num_iterations'. Frequency to compute per-iteration - metrics for the training dataset and validation dataset - (if specified). There can be considerable cost to - computing metrics every iteration, especially if the - training dataset is large. This parameter is a way of - controlling the frequency of those computations. - For example, if you specify 5, then metrics will be computed - every 5 iterations as well as at the end of training - after 'num_iterations'. If you use the default, - metrics will be computed only - once after 'num_iterations' have completed. -
- -
warm_start (optional)
-
BOOLEAN, default: FALSE. - Initalize weights with the coefficients - from the last call of the fit - function. If set to TRUE, weights will be - initialized from the model table - generated by the previous training run. - - @note - The warm start feature works based on the name of the - model output table from a previous training run. - When using warm start, do not drop the model output table - or the model output summary table - before calling the fit function, since these are needed to obtain the - weights from the previous run. - If you are not using warm start, the model output table - and the model output table summary must be dropped in - the usual way before calling the training function. -
- -
name (optional)
-
TEXT, default: NULL. - Free text string to identify a name, if desired. -
- -
description (optional)
-
TEXT, default: NULL. - Free text string to provide a description, if desired. -
-
- -Output tables -
- The model table produced by fit contains the following columns: - - - - - - - - - -
model_weightsBYTEA8. Byte array containing the weights of the neural net.
model_archTEXT. A JSON representation of the model architecture - used in training.
- -A summary table named \_summary is also created, which has the following columns: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
source_tableSource table used for training.
modelModel output table produced by training.
independent_varnameIndependent variables column from the original - source table in the image preprocessing step.
dependent_varnameDependent variable column from the original - source table in the image preprocessing step.
model_arch_tableName of the table containing - the model architecture and (optionally) the - initial model weights.
model_idThe id of the model in - the model architecture table used for training.
compile_paramsCompile parameters passed to Keras.
fit_paramsFit parameters passed to Keras.
num_iterationsNumber of iterations of training completed.
validation_tableName of the table containing - the validation dataset (if specified).
metrics_compute_frequencyFrequency that per-iteration metrics are computed - for the training dataset and validation - dataset.
nameName of the training run (free text).
descriptionDescription of the training run (free text).
model_typeGeneral identifier for type of model trained. - Currently says 'madlib_keras'.
model_sizeSize of the model in KB. Models are stored in - 'bytea' data format which is used for binary strings - in PostgreSQL type databases.
start_training_timeTimestamp for start of training.
end_training_timeTimestamp for end of training.
metrics_elapsed_time Array of elapsed time for metric computations as - per the 'metrics_compute_frequency' parameter. - Useful for drawing a curve showing loss, accuracy or - other metrics as a function of time. - For example, if 'metrics_compute_frequency=5' - this would be an array of elapsed time for every 5th - iteration, plus the last iteration.
madlib_versionVersion of MADlib used.
num_classesCount of distinct classes values used.
class_valuesArray of actual class values used.
dependent_vartypeData type of the dependent variable.
normalizing_constantNormalizing constant used from the - image preprocessing step.
metrics_typeMetric specified in the 'compile_params'.
training_metrics_finalFinal value of the training - metric after all iterations have completed. - The metric reported is the one - specified in the 'metrics_type' parameter.
training_loss_finalFinal value of the training loss after all - iterations have completed.
training_metricsArray of training metrics as - per the 'metrics_compute_frequency' parameter. - For example, if 'metrics_compute_frequency=5' - this would be an array of metrics for every 5th - iteration, plus the last iteration.
training_lossArray of training losses as - per the 'metrics_compute_frequency' parameter. - For example, if 'metrics_compute_frequency=5' - this would be an array of losses for every 5th - iteration, plus the last iteration.
validation_metrics_finalFinal value of the validation - metric after all iterations have completed. - The metric reported is the one - specified in the 'metrics_type' parameter.
validation_loss_finalFinal value of the validation loss after all - iterations have completed.
validation_metricsArray of validation metrics as - per the 'metrics_compute_frequency' parameter. - For example, if 'metrics_compute_frequency=5' - this would be an array of metrics for every 5th - iteration, plus the last iteration.
validation_lossArray of validation losses as - per the 'metrics_compute_frequency' parameter. - For example, if 'metrics_compute_frequency=5' - this would be an array of losses for every 5th - iteration, plus the last iteration.
metrics_itersArray indicating the iterations for which - metrics are calculated, as derived from the - parameters 'num_iterations' and 'metrics_compute_frequency'. - For example, if 'num_iterations=5' - and 'metrics_compute_frequency=2', then 'metrics_iters' value - would be {2,4,5} indicating that metrics were computed - at iterations 2, 4 and 5 (at the end). - If 'num_iterations=5' - and 'metrics_compute_frequency=1', then 'metrics_iters' value - would be {1,2,3,4,5} indicating that metrics were computed - at every iteration.
- -@anchor keras_evaluate -@par Evaluate -The evaluation function has the following format: - -
-madlib_keras_evaluate(
-    model_table,
-    test_table,
-    output_table,
-    use_gpus,
-    mst_key
-    )
-
- -\b Arguments -
- -
model_table
-
TEXT. Name of the table containing the model - to use for validation. -
- -
test_table
-
TEXT. Name of the table containing the evaluation dataset. - Note that test/validation data must be preprocessed in the same - way as the training dataset, so - this is the name of the output - table from the image preprocessor. Independent - and dependent variables are specified in the preprocessor - step which is why you do not need to explictly state - them here as part of the fit function.
- -
output_table
-
TEXT. Name of table that validation output will be - written to. Table contains:
- - - - - - - - - - - - - - -
use_gpus (optional)
-
BOOLEAN, default: FALSE (i.e., CPU). Determines whether GPUs - are to be used for training the neural network. Set to TRUE to use GPUs. - - @note - This parameter must not conflict with how the distribution rules are set in - the preprocessor function. For example, if you set a distribution rule to use - certain segments on hosts that do not have GPUs attached, you will get an error - if you set ‘use_gpus’ to TRUE. Also, we have seen some memory related issues - when segments share GPU resources. - For example, if you have 1 GPU per segment host and your cluster has 4 - segments per segment host, it means that all 4 - segments will share the same - GPU on each host. The current recommended - configuration is 1 GPU per segment. -
- -
mst_key (optional)
-
INTEGER, default: NULL. ID that defines a unique tuple for - model architecture-compile parameters-fit parameters in a model - selection table. Do not use this if training one model at a time using madlib_keras_fit(). - See the Model Selection section - for more details on model selection by training multiple models at a time. -
- - -@anchor keras_predict -@par Predict -The prediction function has the following format: -
-madlib_keras_predict(
-    model_table,
-    test_table,
-    id_col,
-    independent_varname,
-    output_table,
-    pred_type,
-    use_gpus,
-    mst_key
-    )
-
- -\b Arguments -
- -
model_table
-
TEXT. Name of the table containing the model - to use for prediction. -
- -
test_table
-
TEXT. Name of the table containing the dataset to - predict on. Note that test data is not preprocessed (unlike - fit and evaluate) so put one test image per row for prediction. - Also see the comment below for the 'independent_varname' parameter - regarding normalization. - -
- -
id_col
-
TEXT. Name of the id column in the test data table. -
- -
independent_varname
-
TEXT. Column with independent variables in the test table. - If a 'normalizing_const' is specified when preprocessing the - training dataset, this same normalization will be applied to - the independent variables used in predict. -
- -
output_table
-
TEXT. Name of the table that prediction output will be - written to. Table contains:
-
lossLoss value on evaluation dataset.
metricMetric value on evaluation dataset, where 'metrics_type' - below identifies the type of metric.
metrics_typeType of metric used that was used in the training step.
- - - - - - - - - - - - - -
pred_type (optional)
-
TEXT, default: 'response'. The type of output - desired, where 'response' gives the actual prediction - and 'prob' gives the probability value for each class. -
- -
use_gpus (optional)
-
BOOLEAN, default: FALSE (i.e., CPU). - Flag to enable GPU support for training neural network. - The number of GPUs to use is determined by the parameters - passed to the preprocessor. - - @note - We have seen some memory related issues when segments - share GPU resources. - For example, if you provide 1 GPU and your - database cluster is set up to have 4 - segments per segment host, it means that all 4 - segments on a segment host will share the same - GPU. The current recommended - configuration is 1 GPU per segment. -
- -
mst_key (optional)
-
INTEGER, default: NULL. ID that defines a unique tuple for - model architecture-compile parameters-fit parameters in a model - selection table. Do not use this if training one model at a time using madlib_keras_fit(). - See the Model Selection section - for more details on model selection by training multiple models at a time. -
- - - -@anchor keras_predict_byom -@par Predict BYOM (bring your own model) -The predict BYOM function allows you to do inference on models that -have not been trained on MADlib, but rather imported from elsewhere. -It has the following format: -
-madlib_keras_predict_byom(
-    model_arch_table,
-    model_id,
-    test_table,
-    id_col,
-    independent_varname,
-    output_table,
-    pred_type,
-    use_gpus,
-    class_values,
-    normalizing_const
-    )
-
- - -\b Arguments -
- -
model_arch_table
-
TEXT. Name of the architecture table containing the model - to use for prediction. The model weights and architecture can be loaded to - this table by using the - load_keras_model function. -
- -
model_id
-
INTEGER. This is the id in 'model_arch_table' containing the model - architecture and model weights to use for prediction. -
- -
test_table
-
TEXT. Name of the table containing the dataset to - predict on. Note that test data is not preprocessed (unlike - fit and evaluate) so put one test image per row for prediction. - Set the 'normalizing_const' below for the independent variable if necessary. -
- -
id_col
-
TEXT. Name of the id column in the test data table. -
- -
independent_varname
-
TEXT. Column with independent variables in the test table. - Set the 'normalizing_const' below if necessary. -
- -
output_table
-
TEXT. Name of the table that prediction output will be - written to. Table contains:
-
idGives the 'id' for each prediction, corresponding to each row from the test_table.
estimated_COL_NAME - (For pred_type='response') The estimated class - for classification, where - COL_NAME is the name of the column to be - predicted from test data. -
prob_CLASS - (For pred_type='prob' for classification) The - probability of a given class. - There will be one column for each class - in the training data. -
- - - - - - - - - - - - - -
pred_type (optional)
-
TEXT, default: 'response'. The type of output desired, where 'response' - gives the actual prediction and 'prob' gives the probability value for each class. -
- -
use_gpus (optional)
-
BOOLEAN, default: FALSE (i.e., CPU). - Flag to enable GPU support for training neural network. - The number of GPUs to use is determined by the parameters - passed to the preprocessor. - - @note - We have seen some memory related issues when segments - share GPU resources. - For example, if you provide 1 GPU and your - database cluster is set up to have 4 - segments per segment host, it means that all 4 - segments on a segment host will share the same - GPU. The current recommended - configuration is 1 GPU per segment. -
- -
class_values (optional)
-
TEXT[], default: NULL. - List of class labels that were used while training the model. See the 'output_table' - column above for more details. - - @note - If you specify the class values parameter, - it must reflect how the dependent variable was 1-hot encoded for training. If you accidently - pick another order that does not match the 1-hot encoding, the predictions would be wrong. -
- -
normalizing_const (optional)
-
DOUBLE PRECISION, default: 1.0. - The normalizing constant to divide each value in the 'independent_varname' - array by. For example, you would use 255 for this value if the image data is - in the form 0-255. -
- - - -@anchor example -@par Examples - -@note -Deep learning works best on very large datasets, -but that is not convenient for a quick introduction -to the syntax. So in this example we use an MLP on the well -known iris data set from https://archive.ics.uci.edu/ml/datasets/iris. -For more realistic examples with images please refer -to the deep learning notebooks -at https://github.com/apache/madlib-site/tree/asf-site/community-artifacts. - -

Classification

- --# Create an input data set. -
-DROP TABLE IF EXISTS iris_data;
-CREATE TABLE iris_data(
-    id serial,
-    attributes numeric[],
-    class_text varchar
-);
-INSERT INTO iris_data(id, attributes, class_text) VALUES
-(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa'),
-(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa'),
-(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa'),
-(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa'),
-(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa'),
-(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa'),
-(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa'),
-(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa'),
-(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa'),
-(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(11,ARRAY[5.4,3.7,1.5,0.2],'Iris-setosa'),
-(12,ARRAY[4.8,3.4,1.6,0.2],'Iris-setosa'),
-(13,ARRAY[4.8,3.0,1.4,0.1],'Iris-setosa'),
-(14,ARRAY[4.3,3.0,1.1,0.1],'Iris-setosa'),
-(15,ARRAY[5.8,4.0,1.2,0.2],'Iris-setosa'),
-(16,ARRAY[5.7,4.4,1.5,0.4],'Iris-setosa'),
-(17,ARRAY[5.4,3.9,1.3,0.4],'Iris-setosa'),
-(18,ARRAY[5.1,3.5,1.4,0.3],'Iris-setosa'),
-(19,ARRAY[5.7,3.8,1.7,0.3],'Iris-setosa'),
-(20,ARRAY[5.1,3.8,1.5,0.3],'Iris-setosa'),
-(21,ARRAY[5.4,3.4,1.7,0.2],'Iris-setosa'),
-(22,ARRAY[5.1,3.7,1.5,0.4],'Iris-setosa'),
-(23,ARRAY[4.6,3.6,1.0,0.2],'Iris-setosa'),
-(24,ARRAY[5.1,3.3,1.7,0.5],'Iris-setosa'),
-(25,ARRAY[4.8,3.4,1.9,0.2],'Iris-setosa'),
-(26,ARRAY[5.0,3.0,1.6,0.2],'Iris-setosa'),
-(27,ARRAY[5.0,3.4,1.6,0.4],'Iris-setosa'),
-(28,ARRAY[5.2,3.5,1.5,0.2],'Iris-setosa'),
-(29,ARRAY[5.2,3.4,1.4,0.2],'Iris-setosa'),
-(30,ARRAY[4.7,3.2,1.6,0.2],'Iris-setosa'),
-(31,ARRAY[4.8,3.1,1.6,0.2],'Iris-setosa'),
-(32,ARRAY[5.4,3.4,1.5,0.4],'Iris-setosa'),
-(33,ARRAY[5.2,4.1,1.5,0.1],'Iris-setosa'),
-(34,ARRAY[5.5,4.2,1.4,0.2],'Iris-setosa'),
-(35,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(36,ARRAY[5.0,3.2,1.2,0.2],'Iris-setosa'),
-(37,ARRAY[5.5,3.5,1.3,0.2],'Iris-setosa'),
-(38,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(39,ARRAY[4.4,3.0,1.3,0.2],'Iris-setosa'),
-(40,ARRAY[5.1,3.4,1.5,0.2],'Iris-setosa'),
-(41,ARRAY[5.0,3.5,1.3,0.3],'Iris-setosa'),
-(42,ARRAY[4.5,2.3,1.3,0.3],'Iris-setosa'),
-(43,ARRAY[4.4,3.2,1.3,0.2],'Iris-setosa'),
-(44,ARRAY[5.0,3.5,1.6,0.6],'Iris-setosa'),
-(45,ARRAY[5.1,3.8,1.9,0.4],'Iris-setosa'),
-(46,ARRAY[4.8,3.0,1.4,0.3],'Iris-setosa'),
-(47,ARRAY[5.1,3.8,1.6,0.2],'Iris-setosa'),
-(48,ARRAY[4.6,3.2,1.4,0.2],'Iris-setosa'),
-(49,ARRAY[5.3,3.7,1.5,0.2],'Iris-setosa'),
-(50,ARRAY[5.0,3.3,1.4,0.2],'Iris-setosa'),
-(51,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor'),
-(52,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor'),
-(53,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor'),
-(54,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor'),
-(55,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor'),
-(56,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor'),
-(57,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor'),
-(58,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor'),
-(59,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor'),
-(60,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor'),
-(61,ARRAY[5.0,2.0,3.5,1.0],'Iris-versicolor'),
-(62,ARRAY[5.9,3.0,4.2,1.5],'Iris-versicolor'),
-(63,ARRAY[6.0,2.2,4.0,1.0],'Iris-versicolor'),
-(64,ARRAY[6.1,2.9,4.7,1.4],'Iris-versicolor'),
-(65,ARRAY[5.6,2.9,3.6,1.3],'Iris-versicolor'),
-(66,ARRAY[6.7,3.1,4.4,1.4],'Iris-versicolor'),
-(67,ARRAY[5.6,3.0,4.5,1.5],'Iris-versicolor'),
-(68,ARRAY[5.8,2.7,4.1,1.0],'Iris-versicolor'),
-(69,ARRAY[6.2,2.2,4.5,1.5],'Iris-versicolor'),
-(70,ARRAY[5.6,2.5,3.9,1.1],'Iris-versicolor'),
-(71,ARRAY[5.9,3.2,4.8,1.8],'Iris-versicolor'),
-(72,ARRAY[6.1,2.8,4.0,1.3],'Iris-versicolor'),
-(73,ARRAY[6.3,2.5,4.9,1.5],'Iris-versicolor'),
-(74,ARRAY[6.1,2.8,4.7,1.2],'Iris-versicolor'),
-(75,ARRAY[6.4,2.9,4.3,1.3],'Iris-versicolor'),
-(76,ARRAY[6.6,3.0,4.4,1.4],'Iris-versicolor'),
-(77,ARRAY[6.8,2.8,4.8,1.4],'Iris-versicolor'),
-(78,ARRAY[6.7,3.0,5.0,1.7],'Iris-versicolor'),
-(79,ARRAY[6.0,2.9,4.5,1.5],'Iris-versicolor'),
-(80,ARRAY[5.7,2.6,3.5,1.0],'Iris-versicolor'),
-(81,ARRAY[5.5,2.4,3.8,1.1],'Iris-versicolor'),
-(82,ARRAY[5.5,2.4,3.7,1.0],'Iris-versicolor'),
-(83,ARRAY[5.8,2.7,3.9,1.2],'Iris-versicolor'),
-(84,ARRAY[6.0,2.7,5.1,1.6],'Iris-versicolor'),
-(85,ARRAY[5.4,3.0,4.5,1.5],'Iris-versicolor'),
-(86,ARRAY[6.0,3.4,4.5,1.6],'Iris-versicolor'),
-(87,ARRAY[6.7,3.1,4.7,1.5],'Iris-versicolor'),
-(88,ARRAY[6.3,2.3,4.4,1.3],'Iris-versicolor'),
-(89,ARRAY[5.6,3.0,4.1,1.3],'Iris-versicolor'),
-(90,ARRAY[5.5,2.5,4.0,1.3],'Iris-versicolor'),
-(91,ARRAY[5.5,2.6,4.4,1.2],'Iris-versicolor'),
-(92,ARRAY[6.1,3.0,4.6,1.4],'Iris-versicolor'),
-(93,ARRAY[5.8,2.6,4.0,1.2],'Iris-versicolor'),
-(94,ARRAY[5.0,2.3,3.3,1.0],'Iris-versicolor'),
-(95,ARRAY[5.6,2.7,4.2,1.3],'Iris-versicolor'),
-(96,ARRAY[5.7,3.0,4.2,1.2],'Iris-versicolor'),
-(97,ARRAY[5.7,2.9,4.2,1.3],'Iris-versicolor'),
-(98,ARRAY[6.2,2.9,4.3,1.3],'Iris-versicolor'),
-(99,ARRAY[5.1,2.5,3.0,1.1],'Iris-versicolor'),
-(100,ARRAY[5.7,2.8,4.1,1.3],'Iris-versicolor'),
-(101,ARRAY[6.3,3.3,6.0,2.5],'Iris-virginica'),
-(102,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(103,ARRAY[7.1,3.0,5.9,2.1],'Iris-virginica'),
-(104,ARRAY[6.3,2.9,5.6,1.8],'Iris-virginica'),
-(105,ARRAY[6.5,3.0,5.8,2.2],'Iris-virginica'),
-(106,ARRAY[7.6,3.0,6.6,2.1],'Iris-virginica'),
-(107,ARRAY[4.9,2.5,4.5,1.7],'Iris-virginica'),
-(108,ARRAY[7.3,2.9,6.3,1.8],'Iris-virginica'),
-(109,ARRAY[6.7,2.5,5.8,1.8],'Iris-virginica'),
-(110,ARRAY[7.2,3.6,6.1,2.5],'Iris-virginica'),
-(111,ARRAY[6.5,3.2,5.1,2.0],'Iris-virginica'),
-(112,ARRAY[6.4,2.7,5.3,1.9],'Iris-virginica'),
-(113,ARRAY[6.8,3.0,5.5,2.1],'Iris-virginica'),
-(114,ARRAY[5.7,2.5,5.0,2.0],'Iris-virginica'),
-(115,ARRAY[5.8,2.8,5.1,2.4],'Iris-virginica'),
-(116,ARRAY[6.4,3.2,5.3,2.3],'Iris-virginica'),
-(117,ARRAY[6.5,3.0,5.5,1.8],'Iris-virginica'),
-(118,ARRAY[7.7,3.8,6.7,2.2],'Iris-virginica'),
-(119,ARRAY[7.7,2.6,6.9,2.3],'Iris-virginica'),
-(120,ARRAY[6.0,2.2,5.0,1.5],'Iris-virginica'),
-(121,ARRAY[6.9,3.2,5.7,2.3],'Iris-virginica'),
-(122,ARRAY[5.6,2.8,4.9,2.0],'Iris-virginica'),
-(123,ARRAY[7.7,2.8,6.7,2.0],'Iris-virginica'),
-(124,ARRAY[6.3,2.7,4.9,1.8],'Iris-virginica'),
-(125,ARRAY[6.7,3.3,5.7,2.1],'Iris-virginica'),
-(126,ARRAY[7.2,3.2,6.0,1.8],'Iris-virginica'),
-(127,ARRAY[6.2,2.8,4.8,1.8],'Iris-virginica'),
-(128,ARRAY[6.1,3.0,4.9,1.8],'Iris-virginica'),
-(129,ARRAY[6.4,2.8,5.6,2.1],'Iris-virginica'),
-(130,ARRAY[7.2,3.0,5.8,1.6],'Iris-virginica'),
-(131,ARRAY[7.4,2.8,6.1,1.9],'Iris-virginica'),
-(132,ARRAY[7.9,3.8,6.4,2.0],'Iris-virginica'),
-(133,ARRAY[6.4,2.8,5.6,2.2],'Iris-virginica'),
-(134,ARRAY[6.3,2.8,5.1,1.5],'Iris-virginica'),
-(135,ARRAY[6.1,2.6,5.6,1.4],'Iris-virginica'),
-(136,ARRAY[7.7,3.0,6.1,2.3],'Iris-virginica'),
-(137,ARRAY[6.3,3.4,5.6,2.4],'Iris-virginica'),
-(138,ARRAY[6.4,3.1,5.5,1.8],'Iris-virginica'),
-(139,ARRAY[6.0,3.0,4.8,1.8],'Iris-virginica'),
-(140,ARRAY[6.9,3.1,5.4,2.1],'Iris-virginica'),
-(141,ARRAY[6.7,3.1,5.6,2.4],'Iris-virginica'),
-(142,ARRAY[6.9,3.1,5.1,2.3],'Iris-virginica'),
-(143,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(144,ARRAY[6.8,3.2,5.9,2.3],'Iris-virginica'),
-(145,ARRAY[6.7,3.3,5.7,2.5],'Iris-virginica'),
-(146,ARRAY[6.7,3.0,5.2,2.3],'Iris-virginica'),
-(147,ARRAY[6.3,2.5,5.0,1.9],'Iris-virginica'),
-(148,ARRAY[6.5,3.0,5.2,2.0],'Iris-virginica'),
-(149,ARRAY[6.2,3.4,5.4,2.3],'Iris-virginica'),
-(150,ARRAY[5.9,3.0,5.1,1.8],'Iris-virginica');
-
-Create a test/validation dataset from the training data: -
-DROP TABLE IF EXISTS iris_train, iris_test;
--- Set seed so results are reproducible
-SELECT setseed(0);
-SELECT madlib.train_test_split('iris_data',     -- Source table
-                               'iris',          -- Output table root name
-                                0.8,            -- Train proportion
-                                NULL,           -- Test proportion (0.2)
-                                NULL,           -- Strata definition
-                                NULL,           -- Output all columns
-                                NULL,           -- Sample without replacement
-                                TRUE            -- Separate output tables
-                              );
-SELECT COUNT(*) FROM iris_train;
-
-
- count
-------+
-   120
-
- --# Call the preprocessor for deep learning. For the training dataset: -
-\\x off
-DROP TABLE IF EXISTS iris_train_packed, iris_train_packed_summary;
-SELECT madlib.training_preprocessor_dl('iris_train',         -- Source table
-                                       'iris_train_packed',  -- Output table
-                                       'class_text',         -- Dependent variable
-                                       'attributes'          -- Independent variable
-                                        );
-SELECT * FROM iris_train_packed_summary;
-
-
--[ RECORD 1 ]-------+---------------------------------------------
-source_table        | iris_train
-output_table        | iris_train_packed
-dependent_varname   | class_text
-independent_varname | attributes
-dependent_vartype   | character varying
-class_values        | {Iris-setosa,Iris-versicolor,Iris-virginica}
-buffer_size         | 60
-normalizing_const   | 1.0
-num_classes         | 3
-
-For the validation dataset: -
-DROP TABLE IF EXISTS iris_test_packed, iris_test_packed_summary;
-SELECT madlib.validation_preprocessor_dl('iris_test',          -- Source table
-                                         'iris_test_packed',   -- Output table
-                                         'class_text',         -- Dependent variable
-                                         'attributes',         -- Independent variable
-                                         'iris_train_packed'   -- From training preprocessor step
-                                          );
-SELECT * FROM iris_test_packed_summary;
-
-
--[ RECORD 1 ]-------+---------------------------------------------
-source_table        | iris_test
-output_table        | iris_test_packed
-dependent_varname   | class_text
-independent_varname | attributes
-dependent_vartype   | character varying
-class_values        | {Iris-setosa,Iris-versicolor,Iris-virginica}
-buffer_size         | 15
-normalizing_const   | 1.0
-num_classes         | 3
-
- --# Define and load model architecture. Use Keras to define -the model architecture: -
-import keras
-from keras.models import Sequential
-from keras.layers import Dense
-model_simple = Sequential()
-model_simple.add(Dense(10, activation='relu', input_shape=(4,)))
-model_simple.add(Dense(10, activation='relu'))
-model_simple.add(Dense(3, activation='softmax'))
-model_simple.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type)                 Output Shape              Param #
-=================================================================
-dense_1 (Dense)              (None, 10)                50
-_________________________________________________________________
-dense_2 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_3 (Dense)              (None, 3)                 33
-=================================================================
-Total params: 193
-Trainable params: 193
-Non-trainable params: 0
-\endverbatim
-
-Export the model to JSON: -
-model_simple.to_json()
-
-
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-
-Load into model architecture table: -
-DROP TABLE IF EXISTS model_arch_library;
-SELECT madlib.load_keras_model('model_arch_library',  -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json,  -- JSON blob
-                               NULL,                  -- Weights
-                               'Sophie',              -- Name
-                               'A simple model'       -- Descr
-);
-
- --# Train model and view summary table: -
-DROP TABLE IF EXISTS iris_model, iris_model_summary;
-SELECT madlib.madlib_keras_fit('iris_train_packed',   -- source table
-                               'iris_model',          -- model output table
-                               'model_arch_library',  -- model arch table
-                                1,                    -- model arch id
-                                $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$,  -- compile_params
-                                $$ batch_size=5, epochs=3 $$,  -- fit_params
-                                10                    -- num_iterations
-                              );
-SELECT * FROM iris_model_summary;
-
-
--[ RECORD 1 ]-------------+--------------------------------------------------------------------------
-source_table              | iris_train_packed
-model                     | iris_model
-dependent_varname         | class_text
-independent_varname       | attributes
-model_arch_table          | model_arch_library
-model_id                  | 1
-compile_params            |  loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
-fit_params                |  batch_size=5, epochs=3
-num_iterations            | 10
-validation_table          |
-metrics_compute_frequency | 10
-name                      |
-description               |
-model_type                | madlib_keras
-model_size                | 0.7900390625
-start_training_time       | 2019-06-05 20:55:15.785034
-end_training_time         | 2019-06-05 20:55:25.373035
-metrics_elapsed_time      | {9.58799290657043}
-madlib_version            | 1.17.0
-num_classes               | 3
-class_values              | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype         | character varying
-normalizing_const         | 1
-metrics_type              | {accuracy}
-training_metrics_final    | 0.766666650772
-training_loss_final       | 0.721103310585
-training_metrics          | {0.766666650772095}
-training_loss             | {0.721103310585022}
-validation_metrics_final  |
-validation_loss_final     |
-validation_metrics        |
-validation_loss           |
-metrics_iters             | {10}
-
- --# Use the test dataset to evaluate the model we built above: -
-DROP TABLE IF EXISTS iris_validate;
-SELECT madlib.madlib_keras_evaluate('iris_model',       -- model
-                                   'iris_test_packed',  -- test table
-                                   'iris_validate'      -- output table
-                                   );
-SELECT * FROM iris_validate;
-
-
-       loss        |      metric       | metrics_type
--------------------+-------------------+--------------
- 0.719491899013519 | 0.800000011920929 | {accuracy}
-(1 row)
-
- --# Predict. We will use the validation dataset for prediction -as well, which is not usual but serves to show the -syntax. The prediction is in the 'estimated_class_text' -column: -
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib.madlib_keras_predict('iris_model', -- model
-                                   'iris_test',  -- test_table
-                                   'id',  -- id column
-                                   'attributes', -- independent var
-                                   'iris_predict'  -- output table
-                                   );
-SELECT * FROM iris_predict ORDER BY id;
-
-
- id  | estimated_class_text
------+----------------------
-   4 | Iris-setosa
-   6 | Iris-setosa
-   8 | Iris-setosa
-  12 | Iris-setosa
-  13 | Iris-setosa
-  15 | Iris-setosa
-  24 | Iris-setosa
-  30 | Iris-setosa
-  38 | Iris-setosa
-  49 | Iris-setosa
-  60 | Iris-virginica
-  68 | Iris-versicolor
-  69 | Iris-versicolor
-  76 | Iris-versicolor
-  78 | Iris-versicolor
-  81 | Iris-versicolor
-  85 | Iris-virginica
-  90 | Iris-versicolor
-  91 | Iris-versicolor
-  94 | Iris-virginica
- 104 | Iris-virginica
- 106 | Iris-versicolor
- 107 | Iris-virginica
- 110 | Iris-virginica
- 119 | Iris-versicolor
- 127 | Iris-virginica
- 129 | Iris-virginica
- 134 | Iris-versicolor
- 139 | Iris-virginica
- 144 | Iris-virginica
-(30 rows)
-
-Count missclassifications: -
-SELECT COUNT(*) FROM iris_predict JOIN iris_test USING (id)
-WHERE iris_predict.estimated_class_text != iris_test.class_text;
-
-
- count
--------+
-     6
-(1 row)
-
-Accuracy: -
-SELECT round(count(*)*100/(150*0.2),2) as test_accuracy_percent from
-    (select iris_test.class_text as actual, iris_predict.estimated_class_text as estimated
-     from iris_predict inner join iris_test
-     on iris_test.id=iris_predict.id) q
-WHERE q.actual=q.estimated;
-
-
- test_accuracy_percent
------------------------+
-                 80.00
-(1 row)
-
- --# Predict BYOM. -We will use the validation dataset for prediction -as well, which is not usual but serves to show the -syntax. See load_keras_model -for details on how to load the model architecture and weights. -In this example we will use weights we already have: -
-UPDATE model_arch_library
-SET model_weights = iris_model.model_weights
-FROM iris_model
-WHERE model_arch_library.model_id = 1;
-
-Now train using a model from the model architecture table directly -without referencing the model table from the MADlib training. Note that if you -specify the class values parameter as we do below, it must reflect how the dependent -variable was 1-hot encoded for training. In this example the 'training_preprocessor_dl()' -in Step 2 above encoded in the order {'Iris-setosa', 'Iris-versicolor', 'Iris-virginica'} so -this is the order we pass in the parameter. If we accidently pick another order that does -not match the 1-hot encoding, the predictions would be wrong. -
-DROP TABLE IF EXISTS iris_predict_byom;
-SELECT madlib.madlib_keras_predict_byom('model_arch_library',  -- model arch table
-                                         1,                    -- model arch id
-                                        'iris_test',           -- test_table
-                                        'id',                  -- id column
-                                        'attributes',          -- independent var
-                                        'iris_predict_byom',   -- output table
-                                        'response',            -- prediction type
-                                         FALSE,                -- use GPUs
-                                         ARRAY['Iris-setosa', 'Iris-versicolor', 'Iris-virginica'], -- class values
-                                         1.0                   -- normalizing const
-                                   );
-SELECT * FROM iris_predict_byom ORDER BY id;
-
-The prediction is in the 'estimated_dependent_var' column: -
- id  | estimated_dependent_var
------+----------------------
-   4 | Iris-setosa
-   6 | Iris-setosa
-   8 | Iris-setosa
-  12 | Iris-setosa
-  13 | Iris-setosa
-  15 | Iris-setosa
-  24 | Iris-setosa
-  30 | Iris-setosa
-  38 | Iris-setosa
-  49 | Iris-setosa
-  60 | Iris-virginica
-  68 | Iris-versicolor
-  69 | Iris-versicolor
-  76 | Iris-versicolor
-  78 | Iris-versicolor
-  81 | Iris-versicolor
-  85 | Iris-virginica
-  90 | Iris-versicolor
-  91 | Iris-versicolor
-  94 | Iris-virginica
- 104 | Iris-virginica
- 106 | Iris-versicolor
- 107 | Iris-virginica
- 110 | Iris-virginica
- 119 | Iris-versicolor
- 127 | Iris-virginica
- 129 | Iris-virginica
- 134 | Iris-versicolor
- 139 | Iris-virginica
- 144 | Iris-virginica
-(30 rows)
- 
-Count missclassifications: -
-SELECT COUNT(*) FROM iris_predict_byom JOIN iris_test USING (id)
-WHERE iris_predict_byom.estimated_dependent_var != iris_test.class_text;
-
-
- count
--------+
-     6
-(1 row)
-
-Accuracy: -
-SELECT round(count(*)*100/(150*0.2),2) as test_accuracy_percent from
-    (select iris_test.class_text as actual, iris_predict_byom.estimated_dependent_var as estimated
-     from iris_predict_byom inner join iris_test
-     on iris_test.id=iris_predict_byom.id) q
-WHERE q.actual=q.estimated;
-
-
- test_accuracy_percent
------------------------+
-                 80.00
-(1 row)
-
- - -

Classification with Other Parameters

- --# Validation dataset. Now use a validation dataset -and compute metrics every 3rd iteration using -the 'metrics_compute_frequency' parameter. This can -help reduce run time if you do not need metrics -computed at every iteration. -
-DROP TABLE IF EXISTS iris_model, iris_model_summary;
-SELECT madlib.madlib_keras_fit('iris_train_packed',   -- source table
-                               'iris_model',          -- model output table
-                               'model_arch_library',  -- model arch table
-                                1,                    -- model arch id
-                                $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$,  -- compile_params
-                                $$ batch_size=5, epochs=3 $$,  -- fit_params
-                                10,                   -- num_iterations
-                                FALSE,                -- use GPUs
-                                'iris_test_packed',   -- validation dataset
-                                3,                    -- metrics compute frequency
-                                FALSE,                -- warm start
-                               'Sophie L.',           -- name
-                               'Simple MLP for iris dataset'  -- description
-                              );
-SELECT * FROM iris_model_summary;
-
-
--[ RECORD 1 ]-------------+--------------------------------------------------------------------------
-source_table              | iris_train_packed
-model                     | iris_model
-dependent_varname         | class_text
-independent_varname       | attributes
-model_arch_table          | model_arch_library
-model_id                  | 1
-compile_params            |  loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
-fit_params                |  batch_size=5, epochs=3
-num_iterations            | 10
-validation_table          | iris_test_packed
-metrics_compute_frequency | 3
-name                      | Sophie L.
-description               | Simple MLP for iris dataset
-model_type                | madlib_keras
-model_size                | 0.7900390625
-start_training_time       | 2019-06-05 20:58:23.224629
-end_training_time         | 2019-06-05 20:58:35.477499
-metrics_elapsed_time      | {4.69859290122986,8.2062520980835,10.8104848861694,12.2528700828552}
-madlib_version            | 1.17.0
-num_classes               | 3
-class_values              | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype         | character varying
-normalizing_const         | 1
-metrics_type              | {accuracy}
-training_metrics_final    | 0.941666662693
-training_loss_final       | 0.40586027503
-training_metrics          | {0.699999988079071,0.800000011920929,0.899999976158142,0.941666662693024}
-training_loss             | {0.825238645076752,0.534248650074005,0.427499741315842,0.405860275030136}
-validation_metrics_final  | 0.866666674614
-validation_loss_final     | 0.409001916647
-validation_metrics        | {0.733333349227905,0.733333349227905,0.866666674613953,0.866666674613953}
-validation_loss           | {0.827081918716431,0.536275088787079,0.431326270103455,0.409001916646957}
-metrics_iters             | {3,6,9,10}
-
- --# Predict probabilities for each class: -
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib.madlib_keras_predict('iris_model',      -- model
-                                   'iris_test',       -- test_table
-                                   'id',              -- id column
-                                   'attributes',      -- independent var
-                                   'iris_predict',    -- output table
-                                   'prob'             -- response type
-                                   );
-SELECT * FROM iris_predict ORDER BY id;
-
-
- id  | prob_Iris-setosa | prob_Iris-versicolor | prob_Iris-virginica
------+------------------+----------------------+---------------------
-   4 |        0.9241953 |          0.059390426 |          0.01641435
-   6 |        0.9657151 |           0.02809224 |        0.0061926916
-   8 |        0.9543316 |           0.03670931 |         0.008959154
-  12 |       0.93851465 |          0.048681837 |         0.012803554
-  13 |       0.93832576 |           0.04893658 |         0.012737647
-  15 |       0.98717564 |           0.01091238 |        0.0019119986
-  24 |        0.9240628 |          0.060805064 |         0.015132156
-  30 |       0.92063266 |          0.062279057 |         0.017088294
-  38 |        0.9353765 |          0.051353406 |         0.013270103
-  49 |        0.9709265 |          0.023811856 |         0.005261566
-  60 |      0.034395564 |            0.5260507 |          0.43955377
-  68 |      0.031360663 |           0.53689945 |          0.43173987
-  69 |     0.0098787155 |           0.46121457 |          0.52890676
-  76 |      0.031186827 |            0.5644549 |          0.40435827
-  78 |       0.00982633 |           0.48929632 |           0.5008774
-  81 |       0.03658528 |           0.53248984 |           0.4309249
-  85 |      0.015423619 |           0.48452598 |           0.5000504
-  90 |      0.026857043 |            0.5155698 |          0.45757324
-  91 |      0.013675574 |           0.47155368 |           0.5147708
-  94 |      0.073440716 |            0.5418821 |           0.3846772
- 104 |     0.0021637122 |            0.3680499 |          0.62978643
- 106 |    0.00052832486 |           0.30891812 |           0.6905536
- 107 |      0.007315576 |           0.40949163 |           0.5831927
- 110 |     0.0022259138 |            0.4058138 |          0.59196025
- 119 |    0.00018505375 |           0.24510723 |           0.7547077
- 127 |      0.009542585 |           0.46958733 |          0.52087003
- 129 |     0.0019719477 |           0.36288205 |            0.635146
- 134 |     0.0056418083 |           0.43401477 |          0.56034344
- 139 |       0.01067015 |            0.4755573 |          0.51377255
- 144 |     0.0018909549 |           0.37689638 |           0.6212126
-(30 rows)
-
- --# Warm start. Next, use the warm_start parameter -to continue learning, using the coefficients from -the run above. Note that we don't drop the -model table or model summary table: -
-SELECT madlib.madlib_keras_fit('iris_train_packed',   -- source table
-                               'iris_model',          -- model output table
-                               'model_arch_library',  -- model arch table
-                                1,                    -- model arch id
-                                $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$,  -- compile_params
-                                $$ batch_size=5, epochs=3 $$,  -- fit_params
-                                5,                   -- num_iterations
-                                FALSE,               -- use GPUs
-                                'iris_test_packed',   -- validation dataset
-                                1,                    -- metrics compute frequency
-                                TRUE,                 -- warm start
-                               'Sophie L.',           -- name
-                               'Simple MLP for iris dataset'  -- description
-                              );
-SELECT * FROM iris_model_summary;
-
-
--[ RECORD 1 ]-------------+--------------------------------------------------------------------------------------------
-source_table              | iris_train_packed
-model                     | iris_model
-dependent_varname         | class_text
-independent_varname       | attributes
-model_arch_table          | model_arch_library
-model_id                  | 1
-compile_params            |  loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
-fit_params                |  batch_size=5, epochs=3
-num_iterations            | 5
-validation_table          | iris_test_packed
-metrics_compute_frequency | 1
-name                      | Sophie L.
-description               | Simple MLP for iris dataset
-model_type                | madlib_keras
-model_size                | 0.7900390625
-start_training_time       | 2019-06-05 20:59:43.971792
-end_training_time         | 2019-06-05 20:59:51.654586
-metrics_elapsed_time      | {2.89326310157776,4.14273309707642,5.24781513214111,6.34498596191406,7.68279695510864}
-madlib_version            | 1.17.0
-num_classes               | 3
-class_values              | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype         | character varying
-normalizing_const         | 1
-metrics_type              | {accuracy}
-training_metrics_final    | 0.933333337307
-training_loss_final       | 0.334455043077
-training_metrics          | {0.933333337306976,0.933333337306976,0.975000023841858,0.975000023841858,0.933333337306976}
-training_loss             | {0.386842548847198,0.370587915182114,0.357161343097687,0.344598710536957,0.334455043077469}
-validation_metrics_final  | 0.866666674614
-validation_loss_final     | 0.34414178133
-validation_metrics        | {0.866666674613953,0.866666674613953,0.933333337306976,0.866666674613953,0.866666674613953}
-validation_loss           | {0.391442179679871,0.376414686441422,0.362262904644012,0.351912915706635,0.344141781330109}
-metrics_iters             | {1,2,3,4,5}
-
-Note that the loss and accuracy values pick up from where the previous run left off. - -

Transfer Learning

-Here we want -to start with initial weights from a pre-trained model -rather than training from scratch. We also want to use -a model architecture with the earlier feature layer(s) -frozen to save on training time. The example below is -somewhat contrived but gives you the idea of the steps. - --# Define and load a model architecture with -the 1st hidden layer frozen: - -
-model_transfer = Sequential()
-model_transfer.add(Dense(10, activation='relu', input_shape=(4,), trainable=False))
-model_transfer.add(Dense(10, activation='relu'))
-model_transfer.add(Dense(3, activation='softmax'))
-model_simple.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type)                 Output Shape              Param #
-=================================================================
-dense_1 (Dense)              (None, 10)                50
-_________________________________________________________________
-dense_2 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_3 (Dense)              (None, 3)                 33
-=================================================================
-Total params: 193
-Trainable params: 143
-Non-trainable params: 50
-\endverbatim
-
-Export the model to JSON: -
-model_simple.to_json()
-
-
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": false, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-
-Load into model architecture table: -
-SELECT madlib.load_keras_model('model_arch_library',  -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": false, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json,  -- JSON blob
-                               NULL,                  -- Weights
-                               'Maria',               -- Name
-                               'A transfer model'     -- Descr
-);
-
-Fetch the weights from a previous MADlib run. (Normally -these would be downloaded from a source that trained -the same model architecture on a related dataset.) -
-UPDATE model_arch_library
-SET model_weights = iris_model.model_weights
-FROM iris_model
-WHERE model_arch_library.model_id = 2;
-
-Now train the model using the transfer model and the pre-trained weights: -
-DROP TABLE IF EXISTS iris_model, iris_model_summary;
-SELECT madlib.madlib_keras_fit('iris_train_packed',   -- source table
-                               'iris_model',          -- model output table
-                               'model_arch_library',  -- model arch table
-                                2,                    -- model arch id
-                                $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$,  -- compile_params
-                                $$ batch_size=5, epochs=3 $$,  -- fit_params
-                                10                    -- num_iterations
-                              );
-SELECT * FROM iris_model_summary;
-
-
--[ RECORD 1 ]-------------+--------------------------------------------------------------------------
-source_table              | iris_train_packed
-model                     | iris_model
-dependent_varname         | class_text
-independent_varname       | attributes
-model_arch_table          | model_arch_library
-model_id                  | 2
-compile_params            |  loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']
-fit_params                |  batch_size=5, epochs=3
-num_iterations            | 10
-validation_table          |
-metrics_compute_frequency | 10
-name                      |
-description               |
-model_type                | madlib_keras
-model_size                | 0.7900390625
-start_training_time       | 2019-06-05 21:01:03.998422
-end_training_time         | 2019-06-05 21:01:13.525838
-metrics_elapsed_time      | {9.52741599082947}
-madlib_version            | 1.17.0
-num_classes               | 3
-class_values              | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype         | character varying
-normalizing_const         | 1
-metrics_type              | {accuracy}
-training_metrics_final    | 0.975000023842
-training_loss_final       | 0.245171800256
-training_metrics          | {0.975000023841858}
-training_loss             | {0.245171800255775}
-validation_metrics_final  |
-validation_loss_final     |
-validation_metrics        |
-validation_loss           |
-metrics_iters             | {10}
-
- -@anchor notes -@par Notes - -1. Refer to the deep learning section of the Apache MADlib -wiki [5] for important information including supported libraries -and versions. - -2. Classification is currently supported, not regression. - -3. Reminder about the distinction between warm start and transfer learning. Warm start uses model -state (weights) from the model output table from a previous training run - -set the 'warm_start' parameter to TRUE in the fit function. -Transfer learning uses initial model state (weights) stored in the 'model_arch_table' - in this case set the -'warm_start' parameter to FALSE in the fit function. - -@anchor background -@par Technical Background - -For an introduction to deep learning foundations, including MLP and CNN, -refer to [6]. - -This module trains a single large model across the database cluster -using the bulk synchronous parallel (BSP) approach, with model averaging [7]. - -On the effect of database cluster size: as the database cluster size increases, the per iteration -loss will be higher since the model only sees 1/n of the data, where n is the number of segments. -However, each iteration runs faster than single node because it is only traversing 1/n of the data. -For highly non-convex solution spaces, convergence behavior may diminish as cluster size increases. -Ensure that each segment has sufficient volume of data and examples of each class value. - -Alternatively, to train multiple models at the same time for model -architecture search or hyperparameter tuning, you can -use Model Selection, -which does not do model averaging and hence may have better covergence efficiency. - -@anchor literature -@literature - -@anchor mlp-lit-1 -[1] https://keras.io/ - -[2] https://www.tensorflow.org/ - -[3] "Neural Networks for Machine Learning", Lectures 6a and 6b on mini-batch gradient descent, -Geoffrey Hinton with Nitish Srivastava and Kevin Swersky, -http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf - -[4] https://keras.io/models/model/ - -[5] Deep learning section of Apache MADlib wiki, https://cwiki.apache.org/confluence/display/MADLIB/Deep+Learning - -[6] Deep Learning, Ian Goodfellow, Yoshua Bengio and Aaron Courville, MIT Press, 2016. - -[7] "Resource-Efficient and Reproducible Model Selection on Deep Learning Systems," Supun Nakandala, -Yuhao Zhang, and Arun Kumar, Technical Report, Computer Science and Engineering, University of California, -San Diego https://adalabucsd.github.io/papers/TR_2019_Cerebro.pdf. - -[8] Greenplum Database server configuration parameters https://gpdb.docs.pivotal.io/latest/ref_guide/config_params/guc-list.html - -@anchor related -@par Related Topics - -File madlib_keras.sql_in documenting the training, evaluate and predict functions. - -*/ - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit( - source_table VARCHAR, - model VARCHAR, - model_arch_table VARCHAR, - model_id INTEGER, - compile_params VARCHAR, - fit_params VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR, - metrics_compute_frequency INTEGER, - warm_start BOOLEAN, - name VARCHAR, - description VARCHAR -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras') - from utilities.control import SetGUC - with AOControl(False): - with SetGUC("plan_cache_mode", "force_generic_plan"): - madlib_keras.fit(**globals()) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit( - source_table VARCHAR, - model VARCHAR, - model_arch_table VARCHAR, - model_id INTEGER, - compile_params VARCHAR, - fit_params VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR, - metrics_compute_frequency INTEGER, - warm_start BOOLEAN, - name VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit( - source_table VARCHAR, - model VARCHAR, - model_arch_table VARCHAR, - model_id INTEGER, - compile_params VARCHAR, - fit_params VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR, - metrics_compute_frequency INTEGER, - warm_start BOOLEAN -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit( - source_table VARCHAR, - model VARCHAR, - model_arch_table VARCHAR, - model_id INTEGER, - compile_params VARCHAR, - fit_params VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR, - metrics_compute_frequency INTEGER -) RETURNS VOID AS $$ -SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NULL, NULL, NULL); -$$ LANGUAGE sql VOLATILE - m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit( - source_table VARCHAR, - model VARCHAR, - model_arch_table VARCHAR, - model_id INTEGER, - compile_params VARCHAR, - fit_params VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL, NULL, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit( - source_table VARCHAR, - model VARCHAR, - model_arch_table VARCHAR, - model_id INTEGER, - compile_params VARCHAR, - fit_params VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL, NULL, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit( - source_table VARCHAR, - model VARCHAR, - model_arch_table VARCHAR, - model_id INTEGER, - compile_params VARCHAR, - fit_params VARCHAR, - num_iterations INTEGER -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit($1, $2, $3, $4, $5, $6, $7, FALSE, NULL, NULL, NULL, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_transition( - state BYTEA, - dependent_var BYTEA, - independent_var BYTEA, - dependent_var_shape INTEGER[], - independent_var_shape INTEGER[], - model_architecture TEXT, - compile_params TEXT, - fit_params TEXT, - dist_key INTEGER, - dist_key_mapping INTEGER[], - current_seg_id INTEGER, - segments_per_host INTEGER, - images_per_seg INTEGER[], - use_gpus BOOLEAN, - accessible_gpus_for_seg INTEGER[], - prev_serialized_weights BYTEA, - is_final_iteration BOOLEAN -) RETURNS BYTEA AS $$ -PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras') - return madlib_keras.fit_transition(**globals()) -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_merge( - state1 BYTEA, - state2 BYTEA -) RETURNS BYTEA AS $$ -PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras') - return madlib_keras.fit_merge(**globals()) -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_final( - state BYTEA -) RETURNS BYTEA AS $$ -PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras') - return madlib_keras.fit_final(**globals()) -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fit_step( - BYTEA, - BYTEA, - TEXT, - TEXT, - TEXT, - TEXT, - TEXT, - INTEGER, - INTEGER[], - INTEGER, - INTEGER, - INTEGER[], - BOOLEAN, - INTEGER[], - BYTEA, - BOOLEAN); -CREATE AGGREGATE MADLIB_SCHEMA.fit_step( - /* dep_var */ BYTEA, - /* ind_var */ BYTEA, - /* dep_var_shape */ INTEGER[], - /* ind_var_shape */ INTEGER[], - /* model_architecture */ TEXT, - /* compile_params */ TEXT, - /* fit_params */ TEXT, - /* dist_key */ INTEGER, - /* dist_key_mapping */ INTEGER[], - /* current_seg_id */ INTEGER, - /* segments_per_host */ INTEGER, - /* images_per_seg */ INTEGER[], - /* use_gpus */ BOOLEAN, - /* segments_per_host */ INTEGER[], - /* serialized_weights */ BYTEA, - /* is_final_iteration */ BOOLEAN -)( - STYPE=BYTEA, - SFUNC=MADLIB_SCHEMA.fit_transition, - m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.fit_merge,') - FINALFUNC=MADLIB_SCHEMA.fit_final -); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict( - model_table VARCHAR, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR, - pred_type VARCHAR, - use_gpus BOOLEAN, - mst_key INTEGER -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras_predict') - from utilities.control import SetGUC - with AOControl(False): - with SetGUC("plan_cache_mode", "force_generic_plan"): - madlib_keras_predict.Predict(schema_madlib, - model_table, - test_table, - id_col, - independent_varname, - output_table, - pred_type, - use_gpus, - mst_key) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict( - model_table VARCHAR, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR, - pred_type VARCHAR, - use_gpus BOOLEAN -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_predict($1, $2, $3, $4, $5, $6, $7, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict( - model_table VARCHAR, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR, - pred_type VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_predict($1, $2, $3, $4, $5, $6, FALSE, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict( - model_table VARCHAR, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_predict($1, $2, $3, $4, $5, NULL, FALSE, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_predict( - independent_var REAL[], - model_architecture TEXT, - model_weights BYTEA, - is_response BOOLEAN, - normalizing_const DOUBLE PRECISION, - current_seg_id INTEGER, - seg_ids INTEGER[], - images_per_seg INTEGER[], - use_gpus BOOLEAN, - gpus_per_host INTEGER, - segments_per_host INTEGER -) RETURNS DOUBLE PRECISION[] AS $$ - PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras_predict') - return madlib_keras_predict.internal_keras_predict(**globals()) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -------------------------------------------------------------------------------- -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom( - model_arch_table VARCHAR, - model_id INTEGER, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR, - pred_type VARCHAR, - use_gpus BOOLEAN, - class_values TEXT[], - normalizing_const DOUBLE PRECISION -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras_predict') - from utilities.control import SetGUC - with AOControl(False): - with SetGUC("plan_cache_mode", "force_generic_plan"): - madlib_keras_predict.PredictBYOM(**globals()) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom( - model_arch_table VARCHAR, - model_id INTEGER, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR, - pred_type VARCHAR, - use_gpus BOOLEAN, - class_values TEXT[] -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_predict_byom($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom( - model_arch_table VARCHAR, - model_id INTEGER, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR, - pred_type VARCHAR, - use_gpus BOOLEAN -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_predict_byom($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom( - model_arch_table VARCHAR, - model_id INTEGER, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR, - pred_type VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_predict_byom($1, $2, $3, $4, $5, $6, $7, NULL, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict_byom( - model_arch_table VARCHAR, - model_id INTEGER, - test_table VARCHAR, - id_col VARCHAR, - independent_varname VARCHAR, - output_table VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_predict_byom($1, $2, $3, $4, $5, $6, NULL, NULL, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -------------------------------------------------------------------------------- -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate( - model_table VARCHAR, - test_table VARCHAR, - output_table VARCHAR, - use_gpus BOOLEAN, - mst_key INTEGER -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras') - from utilities.control import SetGUC - with AOControl(False): - with SetGUC("plan_cache_mode", "force_generic_plan"): - madlib_keras.evaluate(**globals()) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate( - model_table VARCHAR, - test_table VARCHAR, - output_table VARCHAR, - use_gpus BOOLEAN -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_evaluate($1, $2, $3, $4, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate( - model_table VARCHAR, - test_table VARCHAR, - output_table VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_evaluate($1, $2, $3, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_transition( - state REAL[3], - dependent_var BYTEA, - independent_var BYTEA, - dependent_var_shape INTEGER[], - independent_var_shape INTEGER[], - model_architecture TEXT, - serialized_weights BYTEA, - compile_params TEXT, - dist_key INTEGER, - dist_key_mapping INTEGER[], - current_seg_id INTEGER, - segments_per_host INTEGER, - images_per_seg INTEGER[], - use_gpus BOOLEAN, - accessible_gpus_for_seg INTEGER[], - is_final_iteration BOOLEAN -) RETURNS REAL[3] AS $$ -PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras') - return madlib_keras.internal_keras_eval_transition(**globals()) -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_merge( - state1 REAL[3], - state2 REAL[3] -) RETURNS REAL[3] AS $$ -PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras') - return madlib_keras.internal_keras_eval_merge(**globals()) -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_keras_eval_final( - state REAL[3] -) RETURNS REAL[2] AS $$ -PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras') - return madlib_keras.internal_keras_eval_final(**globals()) -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.internal_keras_evaluate( - BYTEA, - BYTEA, - INTEGER[], - INTEGER[], - TEXT, - BYTEA, - TEXT, - INTEGER, - INTEGER[], - INTEGER, - INTEGER, - INTEGER[], - BOOLEAN, - INTEGER[], - BOOLEAN); - -CREATE AGGREGATE MADLIB_SCHEMA.internal_keras_evaluate( - /* dependent_var */ BYTEA, - /* independent_var */ BYTEA, - /* dependent_var_shape */ INTEGER[], - /* independent_var_shape */ INTEGER[], - /* model_architecture */ TEXT, - /* model_weights */ BYTEA, - /* compile_params */ TEXT, - /* dist_key */ INTEGER, - /* dist_key_mapping */ INTEGER[], - /* current_seg_id */ INTEGER, - /* segments_per_host */ INTEGER, - /* images_per_seg*/ INTEGER[], - /* use_gpus */ BOOLEAN, - /* accessible_gpus_for_seg */ INTEGER[], - /* is_final_iteration */ BOOLEAN -)( - STYPE=REAL[3], - INITCOND='{0,0,0}', - SFUNC=MADLIB_SCHEMA.internal_keras_eval_transition, - m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.internal_keras_eval_merge,') - FINALFUNC=MADLIB_SCHEMA.internal_keras_eval_final -); - - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit( - message VARCHAR -) RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras') - with AOControl(False): - return madlib_keras.fit_help(**globals()) -$$ LANGUAGE plpythonu IMMUTABLE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit() -RETURNS VARCHAR AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit(''); -$$ LANGUAGE sql IMMUTABLE -m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate( - message VARCHAR -) RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras') - with AOControl(False): - return madlib_keras.evaluate_help(**globals()) -$$ LANGUAGE plpythonu IMMUTABLE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_evaluate() -RETURNS VARCHAR AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_evaluate(''); -$$ LANGUAGE sql IMMUTABLE -m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `'); - - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict( - message VARCHAR -) RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras_predict') - with AOControl(False): - return madlib_keras_predict.predict_help(**globals()) -$$ LANGUAGE plpythonu IMMUTABLE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_predict() -RETURNS VARCHAR AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_predict(''); -$$ LANGUAGE sql IMMUTABLE -m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `'); - diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in deleted file mode 100644 index 9de9774..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in +++ /dev/null @@ -1,671 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import plpy -import time -import sys - -from keras.models import * -from madlib_keras import compute_loss_and_metrics -from madlib_keras import get_initial_weights -from madlib_keras import get_model_arch_weights -from madlib_keras import get_source_summary_table_dict -from madlib_keras import should_compute_metrics_this_iter -from madlib_keras_helper import * -from madlib_keras_model_selection import ModelSelectionSchema -from madlib_keras_validator import * -from madlib_keras_wrapper import * - -from utilities.control import MinWarning -from utilities.control import OptimizerControl -from utilities.control import SetGUC -from utilities.utilities import add_postfix -from utilities.utilities import is_platform_gp6_or_up -from utilities.utilities import unique_string -from utilities.utilities import rotate -from utilities.utilities import madlib_version -from utilities.utilities import is_platform_pg -from utilities.utilities import get_seg_number -from utilities.utilities import get_segments_per_host -from utilities.utilities import rename_table - -import json -from collections import defaultdict -import random -import datetime -mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL -mb_indep_var_col = MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL -dist_key_col = DISTRIBUTION_KEY_COLNAME - -""" -FitMultipleModel: This class implements the Model Hopper technique for -training multiple models in parallel. The goal of this function is to train -multiple different models on the same data with different parameters. -The main advantage of this method over running the existing fit function in a -loop is avoiding inaccuracies caused by the model averaging. The basic idea of -model hopper is simple. Let's assume that there are n segments and c*n model -configurations. We begin with distributing these configs to segments. After -that, each segment trains their c models on the data they have locally for one -iteration. Once we have these models, we move them to a different segment -(hopping) as well as receive a different set of models. Once we have the new -models, we use our segments data to refine them (similar to the warm start -functionality). Once every model hops through every segment, we consider an -iteration complete. - -This method ensures that we don't have to average any model and the -loss&accuracy is very close to the ideal case, where all of the data is in one -segment. - -Note that this function is disabled for Postgres. -""" - -@MinWarning("warning") -class FitMultipleModel(): - def __init__(self, schema_madlib, source_table, model_output_table, - model_selection_table, num_iterations, - use_gpus=False, validation_table=None, - metrics_compute_frequency=None, warm_start=False, name="", - description="", **kwargs): - # set the random seed for visit order/scheduling - random.seed(1) - if is_platform_pg(): - plpy.error( - "DL: Multiple model training is not supported on PostgreSQL.") - self.source_table = source_table - self.validation_table = validation_table - self.model_selection_table = model_selection_table - if self.model_selection_table: - self.model_selection_summary_table = add_postfix(self.model_selection_table, '_summary') - - self.num_iterations = num_iterations - self.metrics_compute_frequency = metrics_compute_frequency - self.name = name - self.description = description - self.module_name = 'madlib_keras_fit_multiple_model' - self.schema_madlib = schema_madlib - self.version = madlib_version(self.schema_madlib) - self.mst_key_col = ModelSelectionSchema.MST_KEY - self.model_id_col = ModelSelectionSchema.MODEL_ID - self.compile_params_col = ModelSelectionSchema.COMPILE_PARAMS - self.fit_params_col = ModelSelectionSchema.FIT_PARAMS - self.model_arch_table_col = ModelSelectionSchema.MODEL_ARCH_TABLE - self.model_weights_col = ModelArchSchema.MODEL_WEIGHTS - self.model_arch_col = ModelArchSchema.MODEL_ARCH - self.train_mst_metric_eval_time = defaultdict(list) - self.train_mst_loss = defaultdict(list) - self.train_mst_metric = defaultdict(list) - self.info_str = "" - self.dep_shape_col = add_postfix(mb_dep_var_col, "_shape") - self.ind_shape_col = add_postfix(mb_indep_var_col, "_shape") - self.use_gpus = use_gpus - self.segments_per_host = get_segments_per_host() - if self.use_gpus: - self.accessible_gpus_for_seg = get_accessible_gpus_for_seg( - self.schema_madlib, self.segments_per_host, self.module_name) - else: - self.accessible_gpus_for_seg = get_seg_number()*[0] - - self.original_model_output_table = model_output_table - if self.original_model_output_table: - self.model_info_table = add_postfix(self.original_model_output_table, '_info') - self.model_summary_table = add_postfix( - self.original_model_output_table, '_summary') - - self.model_output_table = self.original_model_output_table - - """ - For warm start, we need to copy the model output table to a temp table - because we call truncate on the model output table while training. - If the query gets aborted, we need to make sure that the user passed - model output table can be recovered. - """ - self.warm_start = bool(warm_start) - self.warm_start_msts = [] - if self.warm_start: - self.model_output_table = unique_string('initial_model') - - self.fit_validator_train = FitMultipleInputValidator( - self.source_table, self.validation_table, self.original_model_output_table, - self.model_selection_table, self.model_selection_summary_table, - mb_dep_var_col, mb_indep_var_col, self.num_iterations, - self.model_info_table, self.mst_key_col, self.model_arch_table_col, - self.metrics_compute_frequency, self.warm_start, self.use_gpus, - self.accessible_gpus_for_seg) - if self.metrics_compute_frequency is None: - self.metrics_compute_frequency = num_iterations - - - self.msts = self.fit_validator_train.msts - self.model_arch_table = self.fit_validator_train.model_arch_table - self.metrics_iters = [] - - original_cuda_env = None - if CUDA_VISIBLE_DEVICES_KEY in os.environ: - original_cuda_env = os.environ[CUDA_VISIBLE_DEVICES_KEY] - - self.dist_key_mapping, self.images_per_seg_train = \ - get_image_count_per_seg_for_minibatched_data_from_db( - self.source_table) - - if self.validation_table: - self.valid_mst_metric_eval_time = defaultdict(list) - self.valid_mst_loss = defaultdict(list) - self.valid_mst_metric = defaultdict(list) - self.dist_key_mapping_valid, self.images_per_seg_valid = \ - get_image_count_per_seg_for_minibatched_data_from_db( - self.validation_table) - self.mst_weights_tbl = unique_string(desp='mst_weights') - self.mst_current_schedule_tbl = unique_string(desp='mst_current_schedule') - - self.dist_keys = query_dist_keys(self.source_table, dist_key_col) - if len(self.msts) < len(self.dist_keys): - self.msts_for_schedule = self.msts + [None] * \ - (len(self.dist_keys) - len(self.msts)) - else: - self.msts_for_schedule = self.msts - random.shuffle(self.msts_for_schedule) - self.grand_schedule = self.generate_schedule(self.msts_for_schedule) - self.gp_segment_id_col = '0' if is_platform_pg() else GP_SEGMENT_ID_COLNAME - self.unlogged_table = "UNLOGGED" if is_platform_gp6_or_up() else '' - - if self.warm_start: - self.create_model_output_table_warm_start() - else: - self.create_model_output_table() - - self.weights_to_update_tbl = unique_string(desp='weights_to_update') - self.fit_multiple_model() - - # Update and cleanup metadata tables - self.insert_info_table() - self.create_model_summary_table() - if self.warm_start: - self.cleanup_for_warm_start() - reset_cuda_env(original_cuda_env) - - def fit_multiple_model(self): - # WARNING: set orca off to prevent unwanted redistribution - with OptimizerControl(False): - self.start_training_time = datetime.datetime.now() - self.metrics_elapsed_start_time = time.time() - self.train_multiple_model() - self.end_training_time = datetime.datetime.now() - - def cleanup_for_warm_start(self): - """ - 1. drop original model table - 2. rename temp to original - :return: - """ - drop_query = "DROP TABLE IF EXISTS {}".format( - self.original_model_output_table) - plpy.execute(drop_query) - rename_table(self.schema_madlib, self.model_output_table, - self.original_model_output_table) - - def train_multiple_model(self): - total_msts = len(self.msts_for_schedule) - for iter in range(1, self.num_iterations+1): - for mst_idx in range(total_msts): - mst_row = [self.grand_schedule[dist_key][mst_idx] - for dist_key in self.dist_keys] - self.create_mst_schedule_table(mst_row) - self.is_final_training_call = (iter == self.num_iterations and mst_idx == total_msts-1) - if mst_idx == 0: - start_iteration = time.time() - self.run_training(mst_idx) - if mst_idx == (total_msts - 1): - end_iteration = time.time() - self.info_str = "\tTime for training in iteration " \ - "{0}: {1} sec\n".format(iter, - end_iteration - - start_iteration) - if should_compute_metrics_this_iter(iter, - self.metrics_compute_frequency, - self.num_iterations): - self.metrics_iters.append(iter) - self.info_str += "\tTraining set after iteration {0}:".format(iter) - self.evaluate_model(iter, self.source_table, True) - if self.validation_table: - self.evaluate_model(iter, self.validation_table, False) - plpy.info("\n"+self.info_str) - - def evaluate_model(self, epoch, table, is_train): - if is_train: - mst_metric_eval_time = self.train_mst_metric_eval_time - mst_loss = self.train_mst_loss - mst_metric = self.train_mst_metric - seg_ids = self.dist_key_mapping - images_per_seg = self.images_per_seg_train - else: - mst_metric_eval_time = self.valid_mst_metric_eval_time - mst_loss = self.valid_mst_loss - mst_metric = self.valid_mst_metric - seg_ids = self.dist_key_mapping_valid - images_per_seg = self.images_per_seg_valid - self.info_str += "\n\tValidation set after iteration {0}:".format(epoch) - for mst in self.msts: - weights = query_weights(self.model_output_table, self.model_weights_col, - self.mst_key_col, mst[self.mst_key_col]) - model_arch, _ = get_model_arch_weights(self.model_arch_table, mst[self.model_id_col]) - _, metric, loss = compute_loss_and_metrics( - self.schema_madlib, table, "$madlib${0}$madlib$".format( - mst[self.compile_params_col]), - model_arch, - weights, - self.use_gpus, - self.accessible_gpus_for_seg, - seg_ids, - images_per_seg, - [], [], epoch, True) - mst_metric_eval_time[mst[self.mst_key_col]] \ - .append(time.time() - self.metrics_elapsed_start_time) - mst_loss[mst[self.mst_key_col]].append(loss) - mst_metric[mst[self.mst_key_col]].append(metric) - self.info_str += "\n\tmst_key={0}: metric={1}, loss={2}".format(mst[self.mst_key_col], metric, loss) - - def generate_schedule(self, msts): - """ Generate the schedule for models hopping to segments """ - grand_schedule = {} - for index, dist_key in enumerate(self.dist_keys): - grand_schedule[dist_key] = rotate(msts, index) - return grand_schedule - - def create_mst_schedule_table(self, mst_row): - mst_temp_query = """ - CREATE {self.unlogged_table} TABLE {self.mst_current_schedule_tbl} - ({self.model_id_col} INTEGER, - {self.compile_params_col} VARCHAR, - {self.fit_params_col} VARCHAR, - {dist_key_col} INTEGER, - {self.mst_key_col} INTEGER) - """.format(dist_key_col=dist_key_col, **locals()) - plpy.execute(mst_temp_query) - for mst, dist_key in zip(mst_row, self.dist_keys): - if mst: - model_id = mst[self.model_id_col] - compile_params = mst[self.compile_params_col] - fit_params = mst[self.fit_params_col] - mst_key = mst[self.mst_key_col] - else: - model_id = "NULL" - compile_params = "NULL" - fit_params = "NULL" - mst_key = "NULL" - mst_insert_query = """ - INSERT INTO {self.mst_current_schedule_tbl} - VALUES ({model_id}, - $madlib${compile_params}$madlib$, - $madlib${fit_params}$madlib$, - {dist_key}, - {mst_key}) - """.format(**locals()) - plpy.execute(mst_insert_query) - - - def create_model_output_table(self): - output_table_create_query = """ - CREATE TABLE {self.model_output_table} - ({self.mst_key_col} INTEGER PRIMARY KEY, - {self.model_weights_col} BYTEA, - {self.model_arch_col} JSON) - """.format(self=self) - plpy.execute(output_table_create_query) - self.initialize_model_output_and_info() - - def create_model_output_table_warm_start(self): - """ - For warm start, we need to copy the model output table to a temp table - because we call truncate on the model output table while training. - If the query gets aborted, we need to make sure that the user passed - model output table can be recovered. - """ - plpy.execute(""" - CREATE TABLE {self.model_output_table} ( - LIKE {self.original_model_output_table} INCLUDING indexes); - """.format(self=self)) - - plpy.execute("""INSERT INTO {self.model_output_table} - SELECT * FROM {self.original_model_output_table}; - """.format(self=self)) - - plpy.execute(""" DELETE FROM {self.model_output_table} - WHERE {self.mst_key_col} NOT IN ( - SELECT {self.mst_key_col} FROM {self.model_selection_table}) - """.format(self=self)) - self.warm_start_msts = plpy.execute( - """ SELECT array_agg({0}) AS a FROM {1} - """.format(self.mst_key_col, self.model_output_table))[0]['a'] - plpy.execute("DROP TABLE {0}".format(self.model_info_table)) - self.initialize_model_output_and_info() - - def initialize_model_output_and_info(self): - info_table_create_query = """ - CREATE TABLE {self.model_info_table} - ({self.mst_key_col} INTEGER PRIMARY KEY, - {self.model_id_col} INTEGER, - {self.compile_params_col} TEXT, - {self.fit_params_col} TEXT, - model_type TEXT, - model_size DOUBLE PRECISION, - metrics_elapsed_time DOUBLE PRECISION[], - metrics_type TEXT[], - training_metrics_final DOUBLE PRECISION, - training_loss_final DOUBLE PRECISION, - training_metrics DOUBLE PRECISION[], - training_loss DOUBLE PRECISION[], - validation_metrics_final DOUBLE PRECISION, - validation_loss_final DOUBLE PRECISION, - validation_metrics DOUBLE PRECISION[], - validation_loss DOUBLE PRECISION[]) - """.format(self=self) - - plpy.execute(info_table_create_query) - for mst in self.msts: - model_arch, model_weights = get_model_arch_weights(self.model_arch_table, - mst[self.model_id_col]) - - - # If warm start is enabled, weights from transfer learning cannot be - # used, even if a particular model doesn't have warm start weights. - if self.warm_start: - model_weights = None - mst_filter = """ - WHERE {mst_col}={mst_key} - """.format( - mst_col=self.mst_key_col, - mst_key=mst['mst_key'] - ) - - else: - mst_filter = '' - - serialized_weights = get_initial_weights(self.model_output_table, - model_arch, - model_weights, - mst['mst_key'] in self.warm_start_msts, - self.use_gpus, - self.accessible_gpus_for_seg, - mst_filter - ) - model_size = sys.getsizeof(serialized_weights) / 1024.0 - - metrics_list = get_metrics_from_compile_param( - mst[self.compile_params_col]) - is_metrics_specified = True if metrics_list else False - metrics_type = 'ARRAY{0}'.format( - metrics_list) if is_metrics_specified else 'NULL' - - info_table_insert_query = """ - INSERT INTO {self.model_info_table}({self.mst_key_col}, - {self.model_id_col}, {self.compile_params_col}, - {self.fit_params_col}, model_type, model_size, - metrics_type) - VALUES ({mst_key_val}, {model_id}, - $madlib${compile_params}$madlib$, - $madlib${fit_params}$madlib$, '{model_type}', - {model_size}, {metrics_type}) - """.format(self=self, - mst_key_val=mst[self.mst_key_col], - model_id=mst[self.model_id_col], - compile_params=mst[self.compile_params_col], - fit_params=mst[self.fit_params_col], - model_type='madlib_keras', - model_size=model_size, - metrics_type=metrics_type) - plpy.execute(info_table_insert_query) - - if not mst['mst_key'] in self.warm_start_msts: - output_table_insert_query = """ - INSERT INTO {self.model_output_table}( - {self.mst_key_col}, {self.model_weights_col}, - {self.model_arch_col}) - VALUES ({mst_key}, $1, $2) - """.format(self=self, - mst_key=mst[self.mst_key_col]) - output_table_insert_query_prepared = plpy.prepare( - output_table_insert_query, ["bytea", "json"]) - plpy.execute(output_table_insert_query_prepared, [ - serialized_weights, model_arch]) - - def create_model_summary_table(self): - if self.warm_start: - plpy.execute("DROP TABLE {0}".format(self.model_summary_table)) - src_summary_dict = get_source_summary_table_dict(self.fit_validator_train) - class_values = src_summary_dict['class_values'] - class_values_type = src_summary_dict['class_values_type'] - dep_vartype = src_summary_dict['dep_vartype'] - dependent_varname = \ - src_summary_dict['dependent_varname_in_source_table'] - independent_varname = \ - src_summary_dict['independent_varname_in_source_table'] - norm_const = src_summary_dict['norm_const'] - self.validation_table = 'NULL' if self.validation_table is None \ - else '$MAD${0}$MAD$'.format(self.validation_table) - if class_values is None: - num_classes = 'NULL' - else: - num_classes = len(class_values) - name = 'NULL' if self.name is None else '$MAD${0}$MAD$'.format(self.name) - descr = 'NULL' if self.description is None else '$MAD${0}$MAD$'.format(self.description) - metrics_iters = self.metrics_iters if self.metrics_iters else 'NULL' - class_values_colname = CLASS_VALUES_COLNAME - dependent_vartype_colname = DEPENDENT_VARTYPE_COLNAME - normalizing_const_colname = NORMALIZING_CONST_COLNAME - float32_sql_type = FLOAT32_SQL_TYPE - create_query = plpy.prepare(""" - CREATE TABLE {self.model_summary_table} AS - SELECT - $MAD${self.source_table}$MAD$::TEXT AS source_table, - {self.validation_table}::TEXT AS validation_table, - $MAD${self.model_output_table}$MAD$::TEXT AS model, - $MAD${self.model_info_table}$MAD$::TEXT AS model_info, - $MAD${dependent_varname}$MAD$::TEXT AS dependent_varname, - $MAD${independent_varname}$MAD$::TEXT AS independent_varname, - $MAD${self.model_arch_table}$MAD$::TEXT AS model_arch_table, - {self.num_iterations}::INTEGER AS num_iterations, - {self.metrics_compute_frequency}::INTEGER AS metrics_compute_frequency, - {self.warm_start} AS warm_start, - {name}::TEXT AS name, - {descr}::TEXT AS description, - '{self.start_training_time}'::TIMESTAMP AS start_training_time, - '{self.end_training_time}'::TIMESTAMP AS end_training_time, - '{self.version}'::TEXT AS madlib_version, - {num_classes}::INTEGER AS num_classes, - $1 AS {class_values_colname}, - $MAD${dep_vartype}$MAD$::TEXT AS {dependent_vartype_colname}, - {norm_const}::{float32_sql_type} AS {normalizing_const_colname}, - ARRAY{metrics_iters}::INTEGER[] AS metrics_iters - """.format(**locals()), [class_values_type]) - plpy.execute(create_query, [class_values]) - - def update_info_table(self, mst, is_train): - mst_key = mst[self.mst_key_col] - metrics, metrics_final, metrics_elapsed_time = \ - "NULL", "NULL", "NULL" - if is_train: - mst_metric = self.train_mst_metric - mst_metric_eval_time = self.train_mst_metric_eval_time - mst_loss = self.train_mst_loss - else: - mst_metric = self.valid_mst_metric - mst_metric_eval_time = self.valid_mst_metric_eval_time - mst_loss = self.valid_mst_loss - - if mst_key in mst_metric: - metrics = mst_metric[mst_key] - metrics_final = metrics[-1] - metrics_elapsed_time = mst_metric_eval_time[mst_key] - metrics = "ARRAY{}".format(metrics) - metrics_elapsed_time = "ARRAY{}".format(metrics_elapsed_time) - loss = mst_loss[mst_key] - loss_final = loss[-1] - loss = "ARRAY{}".format(loss) - if is_train: - update_query = """ - UPDATE {self.model_info_table} SET - training_metrics_final = {metrics_final}, - training_loss_final = {loss_final}, - metrics_elapsed_time = {metrics_elapsed_time}, - training_metrics = {metrics}, - training_loss = {loss} - WHERE {self.mst_key_col} = {mst_key} - """.format(**locals()) - else: - update_query = """ - UPDATE {self.model_info_table} SET - validation_metrics_final = {metrics_final}, - validation_loss_final = {loss_final}, - metrics_elapsed_time = {metrics_elapsed_time}, - validation_metrics = {metrics}, - validation_loss = {loss} - WHERE {self.mst_key_col} = {mst_key} - """.format(**locals()) - plpy.execute(update_query) - - def insert_info_table(self): - for mst in self.msts: - self.update_info_table(mst, True) - if self.validation_table: - self.update_info_table(mst, False) - - def run_training(self, mst_idx): - # NOTE: In the DL module, we want to avoid CREATING TEMP tables - # (creates a slice which stays until the session is disconnected) - # or minimize writing queries that generate plans with Motions (creating - # multiple slices on segments). - # This is mainly to avoid any GPU memory allocation failures. Since GPU - # memory allocation is tied to the process where it is initialized, failures - # may occur when a newly created slice(process) tries allocating GPU memory - # which is already allocated by a previously created slice(process). - # Therefore we want to have queries that do not add motions and all the - # sub-queries running Keras/tensorflow operations reuse the same slice(process) - # that was used for initializing GPU memory. - use_gpus = self.use_gpus if self.use_gpus else False - mst_weights_query = """ - CREATE {self.unlogged_table} TABLE {self.mst_weights_tbl} AS - SELECT mst_tbl.*, wgh_tbl.{self.model_weights_col}, - model_arch_tbl.{self.model_arch_col} - FROM - {self.mst_current_schedule_tbl} mst_tbl - LEFT JOIN {self.model_output_table} wgh_tbl - ON mst_tbl.{self.mst_key_col} = wgh_tbl.{self.mst_key_col} - LEFT JOIN {self.model_arch_table} model_arch_tbl - ON mst_tbl.{self.model_id_col} = model_arch_tbl.{self.model_id_col} - DISTRIBUTED BY ({dist_key_col}) - """.format(dist_key_col=dist_key_col, - **locals()) - plpy.execute(mst_weights_query) - use_gpus = self.use_gpus if self.use_gpus else False - uda_query = """ - CREATE {self.unlogged_table} TABLE {self.weights_to_update_tbl} AS - SELECT {self.schema_madlib}.fit_step_multiple_model({mb_dep_var_col}, - {mb_indep_var_col}, - {self.dep_shape_col}, - {self.ind_shape_col}, - {self.mst_weights_tbl}.{self.model_arch_col}::TEXT, - {self.mst_weights_tbl}.{self.compile_params_col}::TEXT, - {self.mst_weights_tbl}.{self.fit_params_col}::TEXT, - src.{dist_key_col}, - ARRAY{self.dist_key_mapping}, - src.{self.gp_segment_id_col}, - {self.segments_per_host}, - ARRAY{self.images_per_seg_train}, - {use_gpus}::BOOLEAN, - ARRAY{self.accessible_gpus_for_seg}, - {self.mst_weights_tbl}.{self.model_weights_col}::BYTEA, - {is_final_iteration}::BOOLEAN - )::BYTEA AS {self.model_weights_col}, - {self.mst_weights_tbl}.{self.mst_key_col} AS {self.mst_key_col} - ,src.{dist_key_col} AS {dist_key_col} - FROM {self.source_table} src JOIN {self.mst_weights_tbl} - USING ({dist_key_col}) - WHERE {self.mst_weights_tbl}.{self.mst_key_col} IS NOT NULL - GROUP BY src.{dist_key_col}, {self.mst_weights_tbl}.{self.mst_key_col} - DISTRIBUTED BY({dist_key_col}) - """.format(mb_dep_var_col=mb_dep_var_col, - mb_indep_var_col=mb_indep_var_col, - is_final_iteration=True, - dist_key_col=dist_key_col, - use_gpus=use_gpus, - self=self - ) - plpy.execute(uda_query) - - update_query = """ - UPDATE {self.model_output_table} - SET {self.model_weights_col} = {self.weights_to_update_tbl}.{self.model_weights_col} - FROM {self.weights_to_update_tbl} - WHERE {self.model_output_table}.{self.mst_key_col} = {self.weights_to_update_tbl}.{self.mst_key_col} - """.format(self=self) - plpy.execute(update_query) - - self.truncate_and_drop_tables() - - def truncate_and_drop_tables(self): - """ - Context: UPDATE statements in postgres are not in-place replacements but - the row to be updated is marked for deletion(note that the disk space for - this row doesn't get released until vaccuum is called) and a new row in - inserted. - - This function will clear out the disk space used by the model_output_table - and also drop all the other intermediate tables. - If available, set the `` guc so that the truncate command can release the - disk space. The disk space will be released immediately and hence the - model_output table won't grow in size with each UPDATE statement. - - Without this guc, the disk space won't be released and each - call to the UPDATE statement will keep adding to the disk space. The disk - space will only be released when the query is completed. - - The guc can cause data loss if not used properly. Since truncate will - actually clear the disk space immediately, there is no way to recover to - the state before truncate was called on that table. So this guc should only - be set for intermediate tables and never for tables created outside the - scope of the fit_multiple udf. - - Workflow - 1. Create temp table from model table (including the indexes) - 2. truncate the model table to release disk space - 3. rename temp table to model table so that it can be reused for the next - hop - :return: - """ - - with SetGUC("dev_opt_unsafe_truncate_in_subtransaction", "on"): - temp_model_table = unique_string('updated_model') - unlogged_table = self.unlogged_table if not self.is_final_training_call else '' - plpy.execute(""" - CREATE {unlogged_table} TABLE {temp_model_table} ( LIKE {self.model_output_table} - INCLUDING indexes);""".format(temp_model_table=temp_model_table, - unlogged_table=unlogged_table, - self=self)) - plpy.execute(""" - INSERT INTO {temp_model_table} SELECT * FROM {self.model_output_table}; - TRUNCATE TABLE {self.model_output_table}; - DROP TABLE {self.model_output_table}; - """.format(temp_model_table=temp_model_table, self=self)) - rename_table(self.schema_madlib, temp_model_table, - self.model_output_table) - plpy.execute(""" - TRUNCATE TABLE {self.mst_weights_tbl}, {self.mst_current_schedule_tbl}, - {self.weights_to_update_tbl}; - DROP TABLE IF EXISTS {self.mst_weights_tbl}, {self.mst_current_schedule_tbl}, - {self.weights_to_update_tbl};""".format(self=self)) diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in deleted file mode 100644 index 8d68385..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in +++ /dev/null @@ -1,1555 +0,0 @@ -/* ----------------------------------------------------------------------- *//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * - * @file madlib_keras_model_selection.sql_in - * - * @brief SQL functions for model hopper distributed training - * @date August 2019 - * - * - *//* ----------------------------------------------------------------------- */ - -m4_include(`SQLCommon.m4') - -/** -@addtogroup grp_keras_run_model_selection - -@brief Explore network architectures and hyperparameters by training many models a time. - - - -\warning This MADlib method is still in early stage development. -Interface and implementation are subject to change. - -This module allows you to explore network architectures and -hyperparameters by training many models a time across the -database cluster. The aim is to support efficient empirical comparison of multiple -training configurations. This process is called model selection, -and the implementation here is based on a parallel execution strategy -called model hopper parallelism (MOP) [1,2]. - -Models are designed in Keras [3], which is a high-level neural -network API written in Python. It can run -on top of different backends and the one that is currently -supported by MADlib is TensorFlow [4]. - -The main use case is image classification -using sequential models, which are made up of a -linear stack of layers. This includes multilayer perceptrons (MLPs) -and convolutional neural networks (CNNs). Regression is not -currently supported. - -Before doing model selection in MADlib you will need to run -the mini-batch preprocessor, and create a table with the various models -and hyperparameters to try. - -You can mini-batch the training and evaluation datasets by using the -Preprocessor -for Images which is a utility that prepares image data for -use by models that support mini-batch as an optimization option. -This is a one-time operation and you would only -need to re-run the preprocessor if your input data has changed. -The advantage of using mini-batching is that it -can perform better than stochastic gradient descent -because it uses more than one training example at a time, -typically resulting faster and smoother convergence [5]. -The input preprocessor also sets the distribution rules -for the training data. For example, you may only want -to train models on segments that reside on hosts that are GPU enabled. - -You can set up the models and hyperparameters to try with the -Setup -Model Selection utility to define the unique combinations -of model architectures, compile and fit parameters. - -@note If 'madlib_keras_fit_multiple_model()' is running on GPDB 5 and some versions -of GPDB 6, the database will -keep adding to the disk space (in proportion to model size) and will only -release the disk space once the fit multiple query has completed execution. -This is not the case for GPDB 6.5.0+ where disk space is released during the -fit multiple query. - -@note CUDA GPU memory cannot be released until the process holding it is terminated. -When a MADlib deep learning function is called with GPUs, Greenplum internally -creates a process (called a slice) which calls TensorFlow to do the computation. -This process holds the GPU memory until one of the following two things happen: -query finishes and user logs out of the Postgres client/session; or, -query finishes and user waits for the timeout set by gp_vmem_idle_resource_timeout. -The default value for this timeout is 18 sec [8]. So the recommendation is: -log out/reconnect to the session after every GPU query; or -wait for gp_vmem_idle_resource_timeout before you run another GPU query (you can -also set it to a lower value). - -@anchor keras_fit -@par Fit -The fit (training) function has the following format: - -
-madlib_keras_fit_multiple_model(
-    source_table,
-    model_output_table,
-    model_selection_table,
-    num_iterations,
-    use_gpus,
-    validation_table,
-    metrics_compute_frequency,
-    warm_start,
-    name,
-    description
-    )
-
- -\b Arguments -
-
source_table
-
TEXT. Name of the table containing the training data. - This is the name of the output - table from the image preprocessor. Independent - and dependent variables are specified in the preprocessor - step which is why you do not need to explictly state - them here as part of the fit function.
- -
model_output_table
-
TEXT. Name of the output table containing the - multiple models created. - @note pg_temp is not allowed as an output table schema for fit multiple. - Details of output tables are shown below. -
- -
model_selection_table
-
TEXT. Name of the table containing model selection parameters to be tried. - Here we mean both hyper-parameter tuning and model architecture search. -
- -
num_iterations
-
INTEGER. Number of iterations to train. - - @note - This parameter is different than the number of passes over the dataset, - which is commonly referred to as the number of epochs. Since MADlib operates - in a distributed system, the number of - epochs is actually equal to this parameter 'num_iterations' X 'epochs' as - specified in the Keras fit parameter. -
- -
use_gpus (optional)
-
BOOLEAN, default: FALSE (i.e., CPU). Determines whether GPUs - are to be used for training the neural network. Set to TRUE to use GPUs. - - @note - This parameter must not conflict with how the distribution rules are set in - the preprocessor function. For example, if you set a distribution rule to use - certain segments on hosts that do not have GPUs attached, you will get an error - if you set ‘use_gpus’ to TRUE. Also, we have seen some memory related issues - when segments share GPU resources. - For example, if you have 1 GPU per segment host and your cluster has 4 - segments per segment host, it means that all 4 - segments will share the same - GPU on each host. The current recommended - configuration is 1 GPU per segment. -
- -
validation_table (optional)
-
TEXT, default: none. Name of the table containing - the validation dataset. - Note that the validation dataset must be preprocessed - in the same way as the training dataset, so this - is the name of the output - table from running the image preprocessor on the validation dataset. - Using a validation dataset can mean a - longer training time, depending on its size. - This can be controlled using the 'metrics_compute_frequency' - parameter described below.
- -
metrics_compute_frequency (optional)
-
INTEGER, default: once at the end of training - after 'num_iterations'. Frequency to compute per-iteration - metrics for the training dataset and validation dataset - (if specified). There can be considerable cost to - computing metrics every iteration, especially if the - training dataset is large. This parameter is a way of - controlling the frequency of those computations. - For example, if you specify 5, then metrics will be computed - every 5 iterations as well as at the end of training - after 'num_iterations'. If you use the default, - metrics will be computed only - once after 'num_iterations' have completed. -
- -
warm_start (optional)
-
BOOLEAN, default: FALSE. - Initalize weights with the coefficients - from the last call to the fit - function. If set to TRUE, weights will be - initialized from the model table - generated by the previous training run. - - @note - The warm start feature works based on the name of the - model output table from a previous training run. - When using warm start, do not drop the model output table - or the model output summary table - before calling the fit function, since these are needed to obtain the - weights from the previous run. - If you are not using warm start, the model output table - and the model output table summary must be dropped in - the usual way before calling the training function. -
- -
name (optional)
-
TEXT, default: NULL. - Free text string to identify a name, if desired. -
- -
description (optional)
-
TEXT, default: NULL. - Free text string to provide a description, if desired. -
-
- -Output tables -
- The model output table produced by fit contains the following columns. - There is one row per model as per the rows in the 'model_selection_table': -
idGives the 'id' for each prediction, corresponding to each row from the 'test_table'.
estimated_dependent_var - (For pred_type='response') Estimated class for classification. If - the 'class_values' parameter is passed in as NULL, then we assume that the class - labels are [0,1,2...,n-1] where n-1 is the number of classes in the model - architecture. -
prob_CLASS - (For pred_type='prob' for classification) - Probability of a given class. - If 'class_values' is passed in as NULL, we create one column called - 'prob' which is an array of probabilities for each class. - If 'class_values' is not NULL, then there will be one - column for each class. -
- - - - - - - - - - - - -
mst_keyINTEGER. ID that defines a unique tuple for model architecture-compile parameters-fit parameters, - as defined in the 'model_selection_table'.
model_weightsBYTEA8. Byte array containing the weights of the neural net.
model_archTEXT. A JSON representation of the model architecture - used in training.
- - An info table named \_info is also created, which has the following columns. - There is one row per model as per the rows in the 'model_selection_table': - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
mst_keyINTEGER. ID that defines a unique tuple for model architecture-compile parameters-fit parameters, - as defined in the 'model_selection_table'.
model_idINTEGER. ID that defines model in the 'model_arch_table'.
compile_paramsCompile parameters passed to Keras.
fit_paramsFit parameters passed to Keras.
model_typeGeneral identifier for type of model trained. - Currently says 'madlib_keras'.
model_sizeSize of the model in KB. Models are stored in - 'bytea' data format which is used for binary strings - in PostgreSQL type databases.
metrics_elapsed_time Array of elapsed time for metric computations as - per the 'metrics_compute_frequency' parameter. - Useful for drawing a curve showing loss, accuracy or - other metrics as a function of time. - For example, if 'metrics_compute_frequency=5' - this would be an array of elapsed time for every 5th - iteration, plus the last iteration.
metrics_typeMetric specified in the 'compile_params'.
training_metrics_finalFinal value of the training - metric after all iterations have completed. - The metric reported is the one - specified in the 'metrics_type' parameter.
training_loss_finalFinal value of the training loss after all - iterations have completed.
training_metricsArray of training metrics as - per the 'metrics_compute_frequency' parameter. - For example, if 'metrics_compute_frequency=5' - this would be an array of metrics for every 5th - iteration, plus the last iteration.
training_lossArray of training losses as - per the 'metrics_compute_frequency' parameter. - For example, if 'metrics_compute_frequency=5' - this would be an array of losses for every 5th - iteration, plus the last iteration.
validation_metrics_finalFinal value of the validation - metric after all iterations have completed. - The metric reported is the one - specified in the 'metrics_type' parameter.
validation_loss_finalFinal value of the validation loss after all - iterations have completed.
validation_metricsArray of validation metrics as - per the 'metrics_compute_frequency' parameter. - For example, if 'metrics_compute_frequency=5' - this would be an array of metrics for every 5th - iteration, plus the last iteration.
validation_lossArray of validation losses as - per the 'metrics_compute_frequency' parameter. - For example, if 'metrics_compute_frequency=5' - this would be an array of losses for every 5th - iteration, plus the last iteration.
- - A summary table named \_summary is also created, which has the following columns: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
source_tableSource table used for training.
validation_tableName of the table containing - the validation dataset (if specified).
modelName of the output table containing - the model for each model selection tuple.
model_infoName of the output table containing - the model performance and other info for - each model selection tuple.
dependent_varnameDependent variable column from the original - source table in the image preprocessing step.
independent_varnameIndependent variables column from the original - source table in the image preprocessing step.
model_arch_tableName of the table containing - the model architecture and (optionally) the - initial model weights.
num_iterationsNumber of iterations of training completed.
metrics_compute_frequencyFrequency that per-iteration metrics are computed - for the training dataset and validation - datasets.
warm_startIndicates whether warm start used or not.
nameName of the training run (free text).
descriptionDescription of the training run (free text).
start_training_timeTimestamp for start of training.
end_training_timeTimestamp for end of training.
madlib_versionVersion of MADlib used.
num_classesCount of distinct classes values used.
class_valuesArray of actual class values used.
dependent_vartypeData type of the dependent variable.
normalizing_constantNormalizing constant used from the - image preprocessing step.
metrics_itersArray indicating the iterations for which - metrics are calculated, as derived from the - parameters 'num_iterations' and 'metrics_compute_frequency'. - For example, if 'num_iterations=5' - and 'metrics_compute_frequency=2', then 'metrics_iters' value - would be {2,4,5} indicating that metrics were computed - at iterations 2, 4 and 5 (at the end). - If 'num_iterations=5' - and 'metrics_compute_frequency=1', then 'metrics_iters' value - would be {1,2,3,4,5} indicating that metrics were computed - at every iteration.
- -@anchor keras_evaluate -@par Evaluate -The evaluation function has the following format: - -
-madlib_keras_evaluate(
-    model_table,
-    test_table,
-    output_table,
-    use_gpus
-    )
-
- -\b Arguments -
- -
model_table
-
TEXT. Name of the table containing the model - to use for validation. -
- -
test_table
-
TEXT. Name of the table containing the evaluation dataset. - Note that test/validation data must be preprocessed in the same - way as the training dataset, so - this is the name of the output - table from the image preprocessor. Independent - and dependent variables are specified in the preprocessor - step which is why you do not need to explictly state - them here as part of the fit function.
- -
output_table
-
TEXT. Name of table that validation output will be - written to. Table contains:
- - - - - - - - - - - - - - -
use_gpus (optional)
-
BOOLEAN, default: FALSE (i.e., CPU). Determines whether GPUs - are to be used for training the neural network. Set to TRUE to use GPUs. - - @note - This parameter must not conflict with how the distribution rules are set in - the preprocessor function. For example, if you set a distribution rule to use - certain segments on hosts that do not have GPUs attached, you will get an error - if you set ‘use_gpus’ to TRUE. Also, we have seen some memory related issues - when segments share GPU resources. - For example, if you have 1 GPU per segment host and your cluster has 4 - segments per segment host, it means that all 4 - segments will share the same - GPU on each host. The current recommended - configuration is 1 GPU per segment. -
- - -@anchor keras_predict -@par Predict -The prediction function has the following format: -
-madlib_keras_predict(
-    model_table,
-    test_table,
-    id_col,
-    independent_varname,
-    output_table,
-    pred_type,
-    use_gpus
-    )
-
- -\b Arguments -
- -
model_table
-
TEXT. Name of the table containing the model - to use for prediction. -
- -
test_table
-
TEXT. Name of the table containing the dataset to - predict on. Note that test data is not preprocessed (unlike - fit and evaluate) so put one test image per row for prediction. - Also see the comment below for the 'independent_varname' parameter - regarding normalization. - -
- -
id_col
-
TEXT. Name of the id column in the test data table. -
- -
independent_varname
-
TEXT. Column with independent variables in the test table. - If a 'normalizing_const' is specified when preprocessing the - training dataset, this same normalization will be applied to - the independent variables used in predict. -
- -
output_table
-
TEXT. Name of the table that prediction output will be - written to. Table contains:
-
lossLoss value on evaluation dataset.
metricMetric value on evaluation dataset, where 'metrics_type' - below identifies the type of metric.
metrics_typeType of metric used that was used in the training step.
- - - - - - - - - - - - - -
pred_type (optional)
-
TEXT, default: 'response'. The type of output - desired, where 'response' gives the actual prediction - and 'prob' gives the probability value for each class. -
- -
use_gpus (optional)
-
BOOLEAN, default: FALSE (i.e., CPU). Determines - whether GPUs are to be used for prediction/inference. - Set to TRUE to use GPUs. - - @note - The prediction function uses the whole cluster. If you are using GPUs, it - requires that GPUs are attached to all hosts, and that there are the same number - of GPUs on each host (homogeneous cluster). This is different from the fit() - and evaluate() functions that support GPUs on only some of the hosts (heterogeneous cluster). - Therefore, if you have GPUs only on some of the hosts, or an uneven numbers of GPUs per host, then - set this parameter to FALSE to use CPUs. -
- - -@anchor example -@par Examples - -@note -Deep learning works best on very large datasets, -but that is not convenient for a quick introduction -to the syntax. So in this example we use an MLP on the well -known iris data set from https://archive.ics.uci.edu/ml/datasets/iris. -For more realistic examples with images please refer -to the deep learning notebooks -at https://github.com/apache/madlib-site/tree/asf-site/community-artifacts. - -

Classification

- --# Create an input data set. -
-DROP TABLE IF EXISTS iris_data;
-CREATE TABLE iris_data(
-    id serial,
-    attributes numeric[],
-    class_text varchar
-);
-INSERT INTO iris_data(id, attributes, class_text) VALUES
-(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa'),
-(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa'),
-(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa'),
-(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa'),
-(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa'),
-(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa'),
-(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa'),
-(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa'),
-(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa'),
-(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(11,ARRAY[5.4,3.7,1.5,0.2],'Iris-setosa'),
-(12,ARRAY[4.8,3.4,1.6,0.2],'Iris-setosa'),
-(13,ARRAY[4.8,3.0,1.4,0.1],'Iris-setosa'),
-(14,ARRAY[4.3,3.0,1.1,0.1],'Iris-setosa'),
-(15,ARRAY[5.8,4.0,1.2,0.2],'Iris-setosa'),
-(16,ARRAY[5.7,4.4,1.5,0.4],'Iris-setosa'),
-(17,ARRAY[5.4,3.9,1.3,0.4],'Iris-setosa'),
-(18,ARRAY[5.1,3.5,1.4,0.3],'Iris-setosa'),
-(19,ARRAY[5.7,3.8,1.7,0.3],'Iris-setosa'),
-(20,ARRAY[5.1,3.8,1.5,0.3],'Iris-setosa'),
-(21,ARRAY[5.4,3.4,1.7,0.2],'Iris-setosa'),
-(22,ARRAY[5.1,3.7,1.5,0.4],'Iris-setosa'),
-(23,ARRAY[4.6,3.6,1.0,0.2],'Iris-setosa'),
-(24,ARRAY[5.1,3.3,1.7,0.5],'Iris-setosa'),
-(25,ARRAY[4.8,3.4,1.9,0.2],'Iris-setosa'),
-(26,ARRAY[5.0,3.0,1.6,0.2],'Iris-setosa'),
-(27,ARRAY[5.0,3.4,1.6,0.4],'Iris-setosa'),
-(28,ARRAY[5.2,3.5,1.5,0.2],'Iris-setosa'),
-(29,ARRAY[5.2,3.4,1.4,0.2],'Iris-setosa'),
-(30,ARRAY[4.7,3.2,1.6,0.2],'Iris-setosa'),
-(31,ARRAY[4.8,3.1,1.6,0.2],'Iris-setosa'),
-(32,ARRAY[5.4,3.4,1.5,0.4],'Iris-setosa'),
-(33,ARRAY[5.2,4.1,1.5,0.1],'Iris-setosa'),
-(34,ARRAY[5.5,4.2,1.4,0.2],'Iris-setosa'),
-(35,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(36,ARRAY[5.0,3.2,1.2,0.2],'Iris-setosa'),
-(37,ARRAY[5.5,3.5,1.3,0.2],'Iris-setosa'),
-(38,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'),
-(39,ARRAY[4.4,3.0,1.3,0.2],'Iris-setosa'),
-(40,ARRAY[5.1,3.4,1.5,0.2],'Iris-setosa'),
-(41,ARRAY[5.0,3.5,1.3,0.3],'Iris-setosa'),
-(42,ARRAY[4.5,2.3,1.3,0.3],'Iris-setosa'),
-(43,ARRAY[4.4,3.2,1.3,0.2],'Iris-setosa'),
-(44,ARRAY[5.0,3.5,1.6,0.6],'Iris-setosa'),
-(45,ARRAY[5.1,3.8,1.9,0.4],'Iris-setosa'),
-(46,ARRAY[4.8,3.0,1.4,0.3],'Iris-setosa'),
-(47,ARRAY[5.1,3.8,1.6,0.2],'Iris-setosa'),
-(48,ARRAY[4.6,3.2,1.4,0.2],'Iris-setosa'),
-(49,ARRAY[5.3,3.7,1.5,0.2],'Iris-setosa'),
-(50,ARRAY[5.0,3.3,1.4,0.2],'Iris-setosa'),
-(51,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor'),
-(52,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor'),
-(53,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor'),
-(54,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor'),
-(55,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor'),
-(56,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor'),
-(57,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor'),
-(58,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor'),
-(59,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor'),
-(60,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor'),
-(61,ARRAY[5.0,2.0,3.5,1.0],'Iris-versicolor'),
-(62,ARRAY[5.9,3.0,4.2,1.5],'Iris-versicolor'),
-(63,ARRAY[6.0,2.2,4.0,1.0],'Iris-versicolor'),
-(64,ARRAY[6.1,2.9,4.7,1.4],'Iris-versicolor'),
-(65,ARRAY[5.6,2.9,3.6,1.3],'Iris-versicolor'),
-(66,ARRAY[6.7,3.1,4.4,1.4],'Iris-versicolor'),
-(67,ARRAY[5.6,3.0,4.5,1.5],'Iris-versicolor'),
-(68,ARRAY[5.8,2.7,4.1,1.0],'Iris-versicolor'),
-(69,ARRAY[6.2,2.2,4.5,1.5],'Iris-versicolor'),
-(70,ARRAY[5.6,2.5,3.9,1.1],'Iris-versicolor'),
-(71,ARRAY[5.9,3.2,4.8,1.8],'Iris-versicolor'),
-(72,ARRAY[6.1,2.8,4.0,1.3],'Iris-versicolor'),
-(73,ARRAY[6.3,2.5,4.9,1.5],'Iris-versicolor'),
-(74,ARRAY[6.1,2.8,4.7,1.2],'Iris-versicolor'),
-(75,ARRAY[6.4,2.9,4.3,1.3],'Iris-versicolor'),
-(76,ARRAY[6.6,3.0,4.4,1.4],'Iris-versicolor'),
-(77,ARRAY[6.8,2.8,4.8,1.4],'Iris-versicolor'),
-(78,ARRAY[6.7,3.0,5.0,1.7],'Iris-versicolor'),
-(79,ARRAY[6.0,2.9,4.5,1.5],'Iris-versicolor'),
-(80,ARRAY[5.7,2.6,3.5,1.0],'Iris-versicolor'),
-(81,ARRAY[5.5,2.4,3.8,1.1],'Iris-versicolor'),
-(82,ARRAY[5.5,2.4,3.7,1.0],'Iris-versicolor'),
-(83,ARRAY[5.8,2.7,3.9,1.2],'Iris-versicolor'),
-(84,ARRAY[6.0,2.7,5.1,1.6],'Iris-versicolor'),
-(85,ARRAY[5.4,3.0,4.5,1.5],'Iris-versicolor'),
-(86,ARRAY[6.0,3.4,4.5,1.6],'Iris-versicolor'),
-(87,ARRAY[6.7,3.1,4.7,1.5],'Iris-versicolor'),
-(88,ARRAY[6.3,2.3,4.4,1.3],'Iris-versicolor'),
-(89,ARRAY[5.6,3.0,4.1,1.3],'Iris-versicolor'),
-(90,ARRAY[5.5,2.5,4.0,1.3],'Iris-versicolor'),
-(91,ARRAY[5.5,2.6,4.4,1.2],'Iris-versicolor'),
-(92,ARRAY[6.1,3.0,4.6,1.4],'Iris-versicolor'),
-(93,ARRAY[5.8,2.6,4.0,1.2],'Iris-versicolor'),
-(94,ARRAY[5.0,2.3,3.3,1.0],'Iris-versicolor'),
-(95,ARRAY[5.6,2.7,4.2,1.3],'Iris-versicolor'),
-(96,ARRAY[5.7,3.0,4.2,1.2],'Iris-versicolor'),
-(97,ARRAY[5.7,2.9,4.2,1.3],'Iris-versicolor'),
-(98,ARRAY[6.2,2.9,4.3,1.3],'Iris-versicolor'),
-(99,ARRAY[5.1,2.5,3.0,1.1],'Iris-versicolor'),
-(100,ARRAY[5.7,2.8,4.1,1.3],'Iris-versicolor'),
-(101,ARRAY[6.3,3.3,6.0,2.5],'Iris-virginica'),
-(102,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(103,ARRAY[7.1,3.0,5.9,2.1],'Iris-virginica'),
-(104,ARRAY[6.3,2.9,5.6,1.8],'Iris-virginica'),
-(105,ARRAY[6.5,3.0,5.8,2.2],'Iris-virginica'),
-(106,ARRAY[7.6,3.0,6.6,2.1],'Iris-virginica'),
-(107,ARRAY[4.9,2.5,4.5,1.7],'Iris-virginica'),
-(108,ARRAY[7.3,2.9,6.3,1.8],'Iris-virginica'),
-(109,ARRAY[6.7,2.5,5.8,1.8],'Iris-virginica'),
-(110,ARRAY[7.2,3.6,6.1,2.5],'Iris-virginica'),
-(111,ARRAY[6.5,3.2,5.1,2.0],'Iris-virginica'),
-(112,ARRAY[6.4,2.7,5.3,1.9],'Iris-virginica'),
-(113,ARRAY[6.8,3.0,5.5,2.1],'Iris-virginica'),
-(114,ARRAY[5.7,2.5,5.0,2.0],'Iris-virginica'),
-(115,ARRAY[5.8,2.8,5.1,2.4],'Iris-virginica'),
-(116,ARRAY[6.4,3.2,5.3,2.3],'Iris-virginica'),
-(117,ARRAY[6.5,3.0,5.5,1.8],'Iris-virginica'),
-(118,ARRAY[7.7,3.8,6.7,2.2],'Iris-virginica'),
-(119,ARRAY[7.7,2.6,6.9,2.3],'Iris-virginica'),
-(120,ARRAY[6.0,2.2,5.0,1.5],'Iris-virginica'),
-(121,ARRAY[6.9,3.2,5.7,2.3],'Iris-virginica'),
-(122,ARRAY[5.6,2.8,4.9,2.0],'Iris-virginica'),
-(123,ARRAY[7.7,2.8,6.7,2.0],'Iris-virginica'),
-(124,ARRAY[6.3,2.7,4.9,1.8],'Iris-virginica'),
-(125,ARRAY[6.7,3.3,5.7,2.1],'Iris-virginica'),
-(126,ARRAY[7.2,3.2,6.0,1.8],'Iris-virginica'),
-(127,ARRAY[6.2,2.8,4.8,1.8],'Iris-virginica'),
-(128,ARRAY[6.1,3.0,4.9,1.8],'Iris-virginica'),
-(129,ARRAY[6.4,2.8,5.6,2.1],'Iris-virginica'),
-(130,ARRAY[7.2,3.0,5.8,1.6],'Iris-virginica'),
-(131,ARRAY[7.4,2.8,6.1,1.9],'Iris-virginica'),
-(132,ARRAY[7.9,3.8,6.4,2.0],'Iris-virginica'),
-(133,ARRAY[6.4,2.8,5.6,2.2],'Iris-virginica'),
-(134,ARRAY[6.3,2.8,5.1,1.5],'Iris-virginica'),
-(135,ARRAY[6.1,2.6,5.6,1.4],'Iris-virginica'),
-(136,ARRAY[7.7,3.0,6.1,2.3],'Iris-virginica'),
-(137,ARRAY[6.3,3.4,5.6,2.4],'Iris-virginica'),
-(138,ARRAY[6.4,3.1,5.5,1.8],'Iris-virginica'),
-(139,ARRAY[6.0,3.0,4.8,1.8],'Iris-virginica'),
-(140,ARRAY[6.9,3.1,5.4,2.1],'Iris-virginica'),
-(141,ARRAY[6.7,3.1,5.6,2.4],'Iris-virginica'),
-(142,ARRAY[6.9,3.1,5.1,2.3],'Iris-virginica'),
-(143,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'),
-(144,ARRAY[6.8,3.2,5.9,2.3],'Iris-virginica'),
-(145,ARRAY[6.7,3.3,5.7,2.5],'Iris-virginica'),
-(146,ARRAY[6.7,3.0,5.2,2.3],'Iris-virginica'),
-(147,ARRAY[6.3,2.5,5.0,1.9],'Iris-virginica'),
-(148,ARRAY[6.5,3.0,5.2,2.0],'Iris-virginica'),
-(149,ARRAY[6.2,3.4,5.4,2.3],'Iris-virginica'),
-(150,ARRAY[5.9,3.0,5.1,1.8],'Iris-virginica');
-
-Create a test/validation dataset from the training data: -
-DROP TABLE IF EXISTS iris_train, iris_test;
--- Set seed so results are reproducible
-SELECT setseed(0);
-SELECT madlib.train_test_split('iris_data',     -- Source table
-                               'iris',          -- Output table root name
-                                0.8,            -- Train proportion
-                                NULL,           -- Test proportion (0.2)
-                                NULL,           -- Strata definition
-                                NULL,           -- Output all columns
-                                NULL,           -- Sample without replacement
-                                TRUE            -- Separate output tables
-                              );
-SELECT COUNT(*) FROM iris_train;
-
-
- count
-------+
-   120
-
- --# Call the preprocessor for deep learning. For the training dataset: -
-\\x on
-DROP TABLE IF EXISTS iris_train_packed, iris_train_packed_summary;
-SELECT madlib.training_preprocessor_dl('iris_train',         -- Source table
-                                       'iris_train_packed',  -- Output table
-                                       'class_text',         -- Dependent variable
-                                       'attributes'          -- Independent variable
-                                        );
-SELECT * FROM iris_train_packed_summary;
-
-
--[ RECORD 1 ]-------+---------------------------------------------
-source_table        | iris_train
-output_table        | iris_train_packed
-dependent_varname   | class_text
-independent_varname | attributes
-dependent_vartype   | character varying
-class_values        | {Iris-setosa,Iris-versicolor,Iris-virginica}
-buffer_size         | 60
-normalizing_const   | 1.0
-num_classes         | 3
-
-For the validation dataset: -
-DROP TABLE IF EXISTS iris_test_packed, iris_test_packed_summary;
-SELECT madlib.validation_preprocessor_dl('iris_test',          -- Source table
-                                         'iris_test_packed',   -- Output table
-                                         'class_text',         -- Dependent variable
-                                         'attributes',         -- Independent variable
-                                         'iris_train_packed'   -- From training preprocessor step
-                                          );
-SELECT * FROM iris_test_packed_summary;
-
-
--[ RECORD 1 ]-------+---------------------------------------------
-source_table        | iris_test
-output_table        | iris_test_packed
-dependent_varname   | class_text
-independent_varname | attributes
-dependent_vartype   | character varying
-class_values        | {Iris-setosa,Iris-versicolor,Iris-virginica}
-buffer_size         | 15
-normalizing_const   | 1.0
-num_classes         | 3
-
- --# Define and load model architecture. Use Keras to define -the model architecture with 1 hidden layer: -
-import keras
-from keras.models import Sequential
-from keras.layers import Dense
-model1 = Sequential()
-model1.add(Dense(10, activation='relu', input_shape=(4,)))
-model1.add(Dense(10, activation='relu'))
-model1.add(Dense(3, activation='softmax'))
-model1.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type)                 Output Shape              Param #
-=================================================================
-dense_1 (Dense)              (None, 10)                50
-_________________________________________________________________
-dense_2 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_3 (Dense)              (None, 3)                 33
-=================================================================
-Total params: 193
-Trainable params: 193
-Non-trainable params: 0
-\endverbatim
-
-Export the model to JSON: -
-model1.to_json()
-
-
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-
-Define model architecture with 2 hidden layers: -
-model2 = Sequential()
-model2.add(Dense(10, activation='relu', input_shape=(4,)))
-model2.add(Dense(10, activation='relu'))
-model2.add(Dense(10, activation='relu'))
-model2.add(Dense(3, activation='softmax'))
-model2.summary()
-\verbatim
-
-Layer (type)                 Output Shape              Param #
-=================================================================
-dense_4 (Dense)              (None, 10)                50
-_________________________________________________________________
-dense_5 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_6 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_7 (Dense)              (None, 3)                 33
-=================================================================
-Total params: 303
-Trainable params: 303
-Non-trainable params: 0
-\endverbatim
-
-Export the model to JSON: -
-model2.to_json()
-
-
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-
-Load into model architecture table: -
-DROP TABLE IF EXISTS model_arch_library;
-SELECT madlib.load_keras_model('model_arch_library',  -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json,         -- JSON blob
-                               NULL,                  -- Weights
-                               'Sophie',              -- Name
-                               'MLP with 1 hidden layer'       -- Descr
-);
-SELECT madlib.load_keras_model('model_arch_library',  -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json,         -- JSON blob
-                               NULL,                  -- Weights
-                               'Maria',               -- Name
-                               'MLP with 2 hidden layers'       -- Descr
-);
-
--# Define model selection tuples and load. Select the model(s) from the model architecture -table that you want to run, along with the compile and fit parameters. Combinations will be -created for the set of model selection parameters will be loaded: -
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT madlib.load_model_selection_table('model_arch_library', -- model architecture table
-                                         'mst_table',          -- model selection table output
-                                          ARRAY[1,2],              -- model ids from model architecture table
-                                          ARRAY[                   -- compile params
-                                              $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$,
-                                              $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy']$$,
-                                              $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
-                                          ],
-                                          ARRAY[                    -- fit params
-                                              $$batch_size=4,epochs=1$$,
-                                              $$batch_size=8,epochs=1$$
-                                          ]
-                                         );
-SELECT * FROM mst_table ORDER BY mst_key;
-
-
- mst_key | model_id |                                 compile_params                                  |      fit_params
----------+----------+---------------------------------------------------------------------------------+-----------------------
-       1 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1
-       2 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1
-       3 |        1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
-       4 |        1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
-       5 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
-       6 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-       7 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1
-       8 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1
-       9 |        2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
-      10 |        2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
-      11 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
-      12 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-(12 rows)
-
-This is the name of the model architecture table that corresponds to the model selection table: -
-SELECT * FROM mst_table_summary;
-
-
-  model_arch_table
---------------------+
- model_arch_library
-
- --# Train multiple models. -
-DROP TABLE IF EXISTS iris_multi_model, iris_multi_model_summary, iris_multi_model_info;
-SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed',    -- source_table
-                                              'iris_multi_model',     -- model_output_table
-                                              'mst_table',            -- model_selection_table
-                                              10,                     -- num_iterations
-                                              FALSE                   -- use gpus
-                                             );
-
-View the model summary: -
-SELECT * FROM iris_multi_model_summary;
-
-
-source_table              | iris_train_packed
-validation_table          |
-model                     | iris_multi_model
-model_info                | iris_multi_model_info
-dependent_varname         | class_text
-independent_varname       | attributes
-model_arch_table          | model_arch_library
-num_iterations            | 10
-metrics_compute_frequency | 10
-warm_start                | f
-name                      |
-description               |
-start_training_time       | 2019-12-16 18:54:33.826414
-end_training_time         | 2019-12-16 18:56:19.106321
-madlib_version            | 1.17.0
-num_classes               | 3
-class_values              | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype         | character varying
-normalizing_const         | 1
-metrics_iters             | {10}
-
-View results for each model: -
-SELECT * FROM iris_multi_model_info ORDER BY training_metrics_final DESC, training_loss_final;
-
-
- mst_key | model_id |                                 compile_params                                  |      fit_params       |  model_type  |  model_size  | metrics_elapsed_time | metrics_type | training_metrics_final | training_loss_final |  training_metrics   |    training_loss    | validation_metrics_final | validation_loss_final | validation_metrics | validation_loss
----------+----------+---------------------------------------------------------------------------------+-----------------------+--------------+--------------+----------------------+--------------+------------------------+---------------------+---------------------+---------------------+--------------------------+-----------------------+--------------------+-----------------
-       9 |        2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {119.42963886261}    | {accuracy}   |         0.983333349228 |       0.07286978513 | {0.983333349227905} | {0.072869785130024} |                          |                       |                    |
-      10 |        2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {118.485460996628}   | {accuracy}   |         0.975000023842 |     0.0798489004374 | {0.975000023841858} | {0.079848900437355} |                          |                       |                    |
-       4 |        1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {118.707404851913}   | {accuracy}   |         0.975000023842 |      0.143356323242 | {0.975000023841858} | {0.143356323242188} |                          |                       |                    |
-      11 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {118.224883794785}   | {accuracy}   |         0.958333313465 |      0.636615753174 | {0.958333313465118} | {0.636615753173828} |                          |                       |                    |
-       2 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {117.732690811157}   | {accuracy}   |         0.925000011921 |      0.161811202765 | {0.925000011920929} | {0.161811202764511} |                          |                       |                    |
-       5 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {120.357484817505}   | {accuracy}   |         0.833333313465 |        0.5542948246 | {0.833333313465118} | {0.55429482460022}  |                          |                       |                    |
-       3 |        1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {118.928852796555}   | {accuracy}   |         0.824999988079 |      0.301002770662 | {0.824999988079071} | {0.301002770662308} |                          |                       |                    |
-       6 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {120.566634893417}   | {accuracy}   |         0.816666662693 |      0.875298440456 | {0.816666662693024} | {0.87529844045639}  |                          |                       |                    |
-      12 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {119.182703018188}   | {accuracy}   |         0.774999976158 |      0.785651266575 | {0.774999976158142} | {0.78565126657486}  |                          |                       |                    |
-       1 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {119.643137931824}   | {accuracy}   |         0.508333325386 |      0.762569189072 | {0.508333325386047} | {0.762569189071655} |                          |                       |                    |
-       7 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {120.15305685997}    | {accuracy}   |         0.333333343267 |       1.09794270992 | {0.333333343267441} | {1.09794270992279}  |                          |                       |                    |
-       8 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {119.911739826202}   | {accuracy}   |         0.333333343267 |       1.10344016552 | {0.333333343267441} | {1.10344016551971}  |                          |                       |                    |
-(12 rows)
-
- --# Evaluate. Now run evaluate using models we built above: -
-DROP TABLE IF EXISTS iris_validate;
-SELECT madlib.madlib_keras_evaluate('iris_multi_model',  -- model
-                                    'iris_test_packed',  -- test table
-                                    'iris_validate',     -- output table
-                                     NULL,               -- use gpus
-                                     3                   -- mst_key to use
-                                   );
-SELECT * FROM iris_validate;
-
-
-       loss        |      metric       | metrics_type
--------------------+-------------------+--------------
- 0.103803977370262 | 0.966666638851166 | {accuracy}
-
- --# Predict. Now predict using one of the models we built. We will use the validation data set -for prediction as well, which is not usual but serves to show the syntax. -The prediction is in the estimated_class_text column: -
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib.madlib_keras_predict('iris_multi_model', -- model
-                                   'iris_test',        -- test_table
-                                   'id',               -- id column
-                                   'attributes',       -- independent var
-                                   'iris_predict',     -- output table
-                                    'response',        -- prediction type
-                                    FALSE,             -- use gpus
-                                    3                  -- mst_key to use
-                                   );
-SELECT * FROM iris_predict ORDER BY id;
-
-
- id  | estimated_class_text
------+----------------------
-   9 | Iris-setosa
-  18 | Iris-setosa
-  22 | Iris-setosa
-  26 | Iris-setosa
-  35 | Iris-setosa
-  38 | Iris-setosa
-  42 | Iris-setosa
-  43 | Iris-setosa
-  45 | Iris-setosa
-  46 | Iris-setosa
-  50 | Iris-setosa
-  53 | Iris-versicolor
-  60 | Iris-versicolor
-  68 | Iris-versicolor
-  77 | Iris-versicolor
-  78 | Iris-versicolor
-  79 | Iris-versicolor
-  81 | Iris-versicolor
-  82 | Iris-versicolor
-  85 | Iris-virginica
-  95 | Iris-versicolor
-  97 | Iris-versicolor
-  98 | Iris-versicolor
- 113 | Iris-virginica
- 117 | Iris-virginica
- 118 | Iris-virginica
- 127 | Iris-virginica
- 136 | Iris-virginica
- 143 | Iris-virginica
- 145 | Iris-virginica
-(30 rows)
-
-Count missclassifications: -
-SELECT COUNT(*) FROM iris_predict JOIN iris_test USING (id)
-WHERE iris_predict.estimated_class_text != iris_test.class_text;
-
-
- count
--------+
-     1
-
-Percent missclassifications: -
-SELECT round(count(*)*100/(150*0.2),2) as test_accuracy_percent from
-    (select iris_test.class_text as actual, iris_predict.estimated_class_text as estimated
-     from iris_predict inner join iris_test
-     on iris_test.id=iris_predict.id) q
-WHERE q.actual=q.estimated;
-
-
- test_accuracy_percent
------------------------+
-                 96.67
-
- -

Classification with Other Parameters

- --# Validation dataset. Now use a validation dataset -and compute metrics every 3rd iteration using -the 'metrics_compute_frequency' parameter. This can -help reduce run time if you do not need metrics -computed at every iteration. -
-DROP TABLE IF EXISTS iris_multi_model, iris_multi_model_summary, iris_multi_model_info;
-SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed',    -- source_table
-                                              'iris_multi_model',     -- model_output_table
-                                              'mst_table',            -- model_selection_table
-                                               10,                     -- num_iterations
-                                               FALSE,                 -- use gpus
-                                              'iris_test_packed',     -- validation dataset
-                                               3,                     -- metrics compute frequency
-                                               FALSE,                 -- warm start
-                                              'Sophie L.',            -- name
-                                              'Model selection for iris dataset'  -- description
-                                             );
-
-View the model summary: -
-SELECT * FROM iris_multi_model_summary;
-
-
-source_table              | iris_train_packed
-validation_table          | iris_test_packed
-model                     | iris_multi_model
-model_info                | iris_multi_model_info
-dependent_varname         | class_text
-independent_varname       | attributes
-model_arch_table          | model_arch_library
-num_iterations            | 10
-metrics_compute_frequency | 3
-warm_start                | f
-name                      | Sophie L.
-description               | Model selection for iris dataset
-start_training_time       | 2019-12-16 19:28:16.219137
-end_training_time         | 2019-12-16 19:30:19.238692
-madlib_version            | 1.17.0
-num_classes               | 3
-class_values              | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype         | character varying
-normalizing_const         | 1
-metrics_iters             | {3,6,9,10}
-
-View results for each model: -
-SELECT * FROM iris_multi_model_info ORDER BY training_metrics_final DESC, training_loss_final;
-
-
- mst_key | model_id |                                 compile_params                                  |      fit_params       |  model_type  |  model_size  |                         metrics_elapsed_time                          | metrics_type | training_metrics_final | training_loss_final |                             training_metrics                              |                               training_loss                               | validation_metrics_final | validation_loss_final |                            validation_metrics                             |                              validation_loss
----------+----------+---------------------------------------------------------------------------------+-----------------------+--------------+--------------+-----------------------------------------------------------------------+--------------+------------------------+---------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------+--------------------------+-----------------------+---------------------------------------------------------------------------+---------------------------------------------------------------------------
-       4 |        1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {37.0420558452606,78.2046208381653,116.242669820786,134.287139892578} | {accuracy}   |         0.975000023842 |      0.165132269263 | {0.75,0.958333313465118,0.958333313465118,0.975000023841858}              | {0.618549585342407,0.319452553987503,0.223872095346451,0.165132269263268} |           0.966666638851 |        0.213689729571 | {0.733333349227905,0.933333337306976,0.933333337306976,0.966666638851166} | {0.683791160583496,0.370491921901703,0.255890935659409,0.213689729571342}
-       2 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {36.3931469917297,77.5780539512634,115.430645942688,133.599857807159} | {accuracy}   |         0.966666638851 |      0.277698725462 | {0.591666638851166,0.966666638851166,0.666666686534882,0.966666638851166} | {0.634598553180695,0.334936827421188,0.615665555000305,0.27769872546196}  |           0.966666638851 |         0.34405490756 | {0.5,0.966666638851166,0.566666662693024,0.966666638851166}               | {0.643225967884064,0.41021603345871,0.805291295051575,0.344054907560349}
-      10 |        2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {36.8482949733734,78.0155048370361,115.83317399025,134.079672813416}  | {accuracy}   |         0.958333313465 |      0.122385449708 | {0.883333325386047,0.941666662693024,0.858333349227905,0.958333313465118} | {0.291894346475601,0.146935686469078,0.270052850246429,0.122385449707508} |           0.933333337307 |        0.181496843696 | {0.766666650772095,0.866666674613953,0.899999976158142,0.933333337306976} | {0.395013928413391,0.245234906673431,0.301119148731232,0.181496843695641}
-       3 |        1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {37.2318170070648,78.3925468921661,116.45490694046,134.491376876831}  | {accuracy}   |         0.941666662693 |      0.193545326591 | {0.966666638851166,0.941666662693024,0.941666662693024,0.941666662693024} | {0.39665362238884,0.213271111249924,0.190151125192642,0.193545326590538}  |           0.933333337307 |        0.151459023356 | {1,0.966666638851166,0.933333337306976,0.933333337306976}                 | {0.464315593242645,0.198051139712334,0.138570576906204,0.151459023356438}
-       9 |        2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {37.6678929328918,78.820240020752,116.939878940582,134.959810972214}  | {accuracy}   |         0.925000011921 |      0.192344605923 | {0.824999988079071,0.774999976158142,0.966666638851166,0.925000011920929} | {0.434513121843338,0.326292037963867,0.131333693861961,0.192344605922699} |           0.899999976158 |        0.209528595209 | {0.800000011920929,0.766666650772095,0.966666638851166,0.899999976158142} | {0.52033931016922,0.344535797834396,0.170280396938324,0.209528595209122}
-       8 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {38.0689258575439,79.4995639324188,117.36315202713,135.380483865738}  | {accuracy}   |         0.866666674614 |      0.390509605408 | {0.691666662693024,0.691666662693024,0.633333325386047,0.866666674613953} | {0.490214675664902,0.444783747196198,0.627961099147797,0.390509605407715} |           0.933333337307 |        0.376114845276 | {0.566666662693024,0.566666662693024,0.533333361148834,0.933333337306976} | {0.575542628765106,0.54660427570343,0.785183191299438,0.376114845275879}
-       5 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {38.474328994751,79.9709329605103,117.766183853149,135.803887844086}  | {accuracy}   |         0.841666638851 |      0.576696753502 | {0.616666674613953,0.699999988079071,0.758333325386047,0.841666638851166} | {0.90448260307312,0.750164151191711,0.616493880748749,0.576696753501892}  |           0.899999976158 |        0.631914675236 | {0.666666686534882,0.699999988079071,0.733333349227905,0.899999976158142} | {0.871200919151306,0.780709445476532,0.665971457958221,0.631914675235748}
-      11 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {36.6214678287506,77.7987759113312,115.631717920303,133.83836388588}  | {accuracy}   |         0.758333325386 |      0.881635427475 | {0.308333337306976,0.316666662693024,0.75,0.758333325386047}              | {1.12997460365295,1.02749967575073,0.923768699169159,0.881635427474976}   |           0.766666650772 |        0.878168046474 | {0.433333337306976,0.433333337306976,0.766666650772095,0.766666650772095} | {1.07487094402313,0.974115014076233,0.916269063949585,0.878168046474457}
-       7 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {38.2849600315094,79.7524738311768,117.580325841904,135.606695890427} | {accuracy}   |         0.691666662693 |      0.444524824619 | {0.908333361148834,0.391666680574417,0.691666662693024,0.691666662693024} | {0.335082054138184,2.02327847480774,0.444351017475128,0.444524824619293}  |           0.566666662693 |        0.539750337601 | {0.800000011920929,0.266666680574417,0.566666662693024,0.566666662693024} | {0.433189332485199,2.3276960849762,0.534160375595093,0.539750337600708}
-       6 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {38.6593668460846,80.1789360046387,117.957875013351,135.995815992355} | {accuracy}   |         0.683333337307 |      0.841839790344 | {0.316666662693024,0.366666674613953,0.666666686534882,0.683333337306976} | {1.07646071910858,0.963329672813416,0.87216705083847,0.841839790344238}   |           0.666666686535 |        0.840192914009 | {0.433333337306976,0.533333361148834,0.666666686534882,0.666666686534882} | {1.02845978736877,0.941896677017212,0.861787617206573,0.840192914009094}
-       1 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {37.8553328514099,79.2480089664459,117.139881849289,135.155915975571} | {accuracy}   |         0.358333319426 |       1.11013436317 | {0.358333319425583,0.333333343267441,0.333333343267441,0.358333319425583} | {1.10554325580597,1.11694586277008,1.09756696224213,1.11013436317444}     |           0.233333334327 |         1.17629003525 | {0.233333334326744,0.333333343267441,0.333333343267441,0.233333334326744} | {1.16081762313843,1.14324629306793,1.11625325679779,1.1762900352478}
-      12 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {37.4500079154968,78.6058378219604,116.700626850128,134.72905087471}  | {accuracy}   |         0.308333337307 |       1.06241953373 | {0.150000005960464,0.333333343267441,0.333333343267441,0.308333337306976} | {1.13338851928711,1.09694564342499,1.07030868530273,1.06241953372955}     |           0.433333337307 |         1.03659796715 | {0.16666667163372,0.333333343267441,0.433333337306976,0.433333337306976}  | {1.06262242794037,1.07252764701843,1.05843663215637,1.03659796714783}
-(12 rows)
-
- --# Predict probabilities for each class: -
-DROP TABLE IF EXISTS iris_predict;
-SELECT madlib.madlib_keras_predict('iris_multi_model', -- model
-                                   'iris_test',        -- test_table
-                                   'id',               -- id column
-                                   'attributes',       -- independent var
-                                   'iris_predict',     -- output table
-                                    'prob',            -- prediction type
-                                    FALSE,             -- use gpus
-                                    3                  -- mst_key to use
-                                   );
-SELECT * FROM iris_predict ORDER BY id;
-
-
- id  | prob_Iris-setosa | prob_Iris-versicolor | prob_Iris-virginica
------+------------------+----------------------+---------------------
-   9 |       0.99931216 |        0.00068789057 |       6.2587335e-10
-  18 |       0.99984336 |        0.00015656587 |        7.969957e-12
-  22 |        0.9998497 |        0.00015029701 |       6.4133347e-12
-  26 |        0.9995004 |        0.00049964694 |       2.2795305e-10
-  35 |       0.99964666 |        0.00035332117 |       9.4490485e-11
-  38 |       0.99964666 |        0.00035332117 |       9.4490485e-11
-  42 |        0.9985154 |         0.0014845316 |        5.293262e-09
-  43 |       0.99964476 |         0.0003552362 |        9.701174e-11
-  45 |        0.9997311 |        0.00026883607 |        3.076166e-11
-  46 |        0.9995486 |        0.00045140853 |       1.6814435e-10
-  50 |        0.9997856 |        0.00021441824 |       2.1316622e-11
-  53 |     9.837335e-06 |           0.97109175 |         0.028898403
-  60 |    0.00014028326 |           0.96552837 |         0.034331344
-  68 |    0.00087942625 |            0.9883348 |         0.010785843
-  77 |      6.08114e-06 |           0.94356424 |         0.056429718
-  78 |     7.116364e-07 |            0.8596206 |          0.14037873
-  79 |    1.3918722e-05 |           0.94052655 |          0.05945957
-  81 |    0.00045687397 |            0.9794796 |         0.020063542
-  82 |     0.0015463434 |           0.98768973 |         0.010763981
-  85 |    1.0929693e-05 |           0.87866926 |         0.121319845
-  95 |    6.3600986e-05 |           0.95264935 |         0.047287125
-  97 |    0.00020298029 |             0.981617 |         0.018180028
-  98 |    0.00019721613 |           0.98902065 |          0.01078211
- 113 |    1.0388683e-09 |           0.23626474 |           0.7637353
- 117 |     4.598902e-09 |           0.25669694 |           0.7433031
- 118 |    3.7139156e-11 |           0.13193987 |           0.8680601
- 127 |    2.1297862e-07 |             0.670349 |          0.32965073
- 136 |    7.1760774e-12 |           0.07074605 |            0.929254
- 143 |    1.2568385e-09 |          0.113820426 |           0.8861796
- 145 |      6.17019e-11 |          0.117578305 |          0.88242173
-(30 rows)
-
- --# Warm start. Next, use the warm_start parameter -to continue learning, using the coefficients from -the run above. Note that we don't drop the -model table or model summary table: -
-SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed',    -- source_table
-                                              'iris_multi_model',     -- model_output_table
-                                              'mst_table',            -- model_selection_table
-                                               3,                     -- num_iterations
-                                               FALSE,                 -- use gpus
-                                              'iris_test_packed',     -- validation dataset
-                                               1,                     -- metrics compute frequency
-                                               TRUE,                  -- warm start
-                                              'Sophie L.',            -- name
-                                              'Simple MLP for iris dataset'  -- description
-                                             );
-SELECT * FROM iris_multi_model_summary;
-
-
-source_table              | iris_train_packed
-validation_table          | iris_test_packed
-model                     | iris_multi_model
-model_info                | iris_multi_model_info
-dependent_varname         | class_text
-independent_varname       | attributes
-model_arch_table          | model_arch_library
-num_iterations            | 3
-metrics_compute_frequency | 1
-warm_start                | t
-name                      | Sophie L.
-description               | Simple MLP for iris dataset
-start_training_time       | 2019-12-16 20:07:41.488587
-end_training_time         | 2019-12-16 20:08:27.20651
-madlib_version            | 1.17.0
-num_classes               | 3
-class_values              | {Iris-setosa,Iris-versicolor,Iris-virginica}
-dependent_vartype         | character varying
-normalizing_const         | 1
-metrics_iters             | {1,2,3}
-
-View results for each model: -
-SELECT * FROM iris_multi_model_info ORDER BY training_metrics_final DESC, training_loss_final;
-
-
- mst_key | model_id |                                 compile_params                                  |      fit_params       |  model_type  |  model_size  |                 metrics_elapsed_time                 | metrics_type | training_metrics_final | training_loss_final |                    training_metrics                     |                      training_loss                       | validation_metrics_final | validation_loss_final |                   validation_metrics                    |                     validation_loss
----------+----------+---------------------------------------------------------------------------------+-----------------------+--------------+--------------+------------------------------------------------------+--------------+------------------------+---------------------+---------------------------------------------------------+----------------------------------------------------------+--------------------------+-----------------------+---------------------------------------------------------+---------------------------------------------------------
-       5 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {19.451014995575,37.2563629150391,54.7182998657227}  | {accuracy}   |         0.975000023842 |      0.490673750639 | {0.958333313465118,0.691666662693024,0.975000023841858} | {0.541427075862885,0.517450392246246,0.490673750638962}  |           0.933333337307 |        0.557333409786 | {0.933333337306976,0.666666686534882,0.933333337306976} | {0.60710871219635,0.570206344127655,0.557333409786224}
-       9 |        2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 0.7900390625 | {18.2973220348358,36.3793680667877,54.0178129673004} | {accuracy}   |         0.966666638851 |     0.0894369781017 | {0.966666638851166,0.966666638851166,0.966666638851166} | {0.133233144879341,0.111788973212242,0.0894369781017303} |           0.899999976158 |        0.195293620229 | {0.933333337306976,0.966666638851166,0.899999976158142} | {0.156044512987137,0.132803827524185,0.195293620228767}
-       4 |        1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {17.6080539226532,35.6788699626923,53.3836889266968} | {accuracy}   |         0.966666638851 |      0.147051945329 | {0.908333361148834,0.958333313465118,0.966666638851166} | {0.225205257534981,0.168186634778976,0.147051945328712}  |           0.866666674614 |        0.250319689512 | {0.899999976158142,0.933333337306976,0.866666674613953} | {0.23467344045639,0.182851999998093,0.250319689512253}
-       8 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {18.7529940605164,36.8255958557129,54.3704080581665} | {accuracy}   |         0.966666638851 |      0.244641214609 | {0.691666662693024,0.891666650772095,0.966666638851166} | {0.939713299274445,0.462556451559067,0.244641214609146}  |           0.966666638851 |        0.298279434443 | {0.566666662693024,0.966666638851166,0.966666638851166} | {1.30671143531799,0.412235885858536,0.29827943444252}
-      10 |        2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {17.4004180431366,35.4556438922882,53.1877279281616} | {accuracy}   |         0.958333313465 |      0.123381219804 | {0.949999988079071,0.766666650772095,0.958333313465118} | {0.0919980704784393,0.576169073581696,0.123381219804287} |           0.933333337307 |        0.203262642026 | {0.866666674613953,0.766666650772095,0.933333337306976} | {0.199721112847328,0.959742486476898,0.203262642025948}
-       3 |        1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {17.81547498703,35.8978669643402,53.5737180709839}   | {accuracy}   |         0.933333337307 |      0.150664463639 | {0.941666662693024,0.925000011920929,0.933333337306976} | {0.117781177163124,0.163000836968422,0.150664463639259}  |           0.833333313465 |        0.365329563618 | {0.866666674613953,0.833333313465118,0.833333313465118} | {0.249404579401016,0.375173389911652,0.365329563617706}
-       6 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {19.686233997345,37.4543249607086,54.8708770275116}  | {accuracy}   |         0.858333349228 |      0.743227303028 | {0.675000011920929,0.708333313465118,0.858333349227905} | {0.808507084846497,0.774080872535706,0.743227303028107}  |           0.966666638851 |        0.770158529282 | {0.666666686534882,0.666666686534882,0.966666638851166} | {0.808504283428192,0.791898012161255,0.770158529281616}
-      11 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {17.1583528518677,35.0312390327454,52.96133685112}   | {accuracy}   |         0.816666662693 |      0.739802956581 | {0.774999976158142,0.816666662693024,0.816666662693024} | {0.83727890253067,0.792884111404419,0.739802956581116}   |           0.800000011921 |        0.758302807808 | {0.766666650772095,0.800000011920929,0.800000011920929} | {0.837629973888397,0.801746726036072,0.758302807807922}
-       2 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1 | madlib_keras | 1.2197265625 | {16.9146749973297,34.794900894165,52.7328250408173}  | {accuracy}   |         0.808333337307 |      0.303489625454 | {0.683333337306976,0.966666638851166,0.808333337306976} | {1.05107569694519,0.189959138631821,0.303489625453949}   |           0.866666674614 |        0.285375326872 | {0.666666686534882,0.966666638851166,0.866666674613953} | {1.01942157745361,0.238933652639389,0.285375326871872}
-      12 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 | madlib_keras | 0.7900390625 | {18.0590150356293,36.1394078731537,53.7930529117584} | {accuracy}   |         0.699999988079 |       1.02253305912 | {0.550000011920929,0.691666662693024,0.699999988079071} | {1.0493084192276,1.03803598880768,1.02253305912018}      |           0.666666686535 |         1.02013540268 | {0.633333325386047,0.600000023841858,0.666666686534882} | {1.03952574729919,1.03439521789551,1.02013540267944}
-       7 |        2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {19.2141709327698,37.0566499233246,54.5629329681396} | {accuracy}   |         0.691666662693 |      0.448221176863 | {0.691666662693024,0.691666662693024,0.691666662693024} | {0.447027385234833,0.444605946540833,0.448221176862717}  |           0.566666662693 |        0.555035352707 | {0.566666662693024,0.566666662693024,0.566666662693024} | {0.551217257976532,0.540408432483673,0.555035352706909}
-       1 |        1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1 | madlib_keras | 1.2197265625 | {18.501914024353,36.5938439369202,54.194118976593}   | {accuracy}   |         0.358333319426 |       1.09730923176 | {0.333333343267441,0.333333343267441,0.358333319425583} | {1.09999334812164,1.10405397415161,1.09730923175812}     |           0.233333334327 |         1.12532019615 | {0.333333343267441,0.333333343267441,0.233333334326744} | {1.12446486949921,1.13782525062561,1.12532019615173}
-(12 rows)
-
-Note that the loss and accuracy values pick up from where the previous run left off. - -@anchor notes -@par Notes - -1. Refer to the deep learning section of the Apache MADlib -wiki [6] for important information including supported libraries -and versions. - -2. Classification is currently supported, not regression. - -3. Reminder about the distinction between warm start and transfer learning. Warm start uses model -state (weights) from the model output table from a previous training run - -set the 'warm_start' parameter to TRUE in the fit function. -Transfer learning uses initial model state (weights) stored in the 'model_arch_table' - in this case set the -'warm_start' parameter to FALSE in the fit function. - -4. Here are some more details on how warm start works. These details are mostly applicable when implementing autoML algorithms on top of MADlib's model selection. In short, the 'model_selection_table' dictates which models get trained and output to the 'model_output_table' and associated summary and info tables. When 'warm_start' is TRUE, models are built for each 'mst_key' in the 'model_selection_table'. If there are prior runs for an 'mst_key' then the weights from that run will be used. If there are no prior runs for an 'mst_key' then random initialization will be used. For example, let's say we start with 'mst_keys' of 1, 2, 3, and 4 in the 'model_selection_table'. We run fit once to get model and info tables for 1, 2, 3, and 4. Then we modify the 'model_selection_table' as part of an autoML scheme, in which we remove the 'mst_key' for 1 and add a new 'mst_key' for 5. Next we run fit with warm start. The result will be models created for 'mst_keys' of 2, 3, 4, and 5. Warm start will be used for 2, 3, and 4 (using prior run) and random initialization will be used for 5 (no prior run). The 'mst_key' of 1 will be dropped. - -5. The 'num_iterations' parameter and the Keras fit parameter 'epochs' can substantially affect accuracy and run-time. -In general, increasing the number of 'epochs' for a fixed 'num_iterations' will speed up training, but may result -in lower accuracy. It's best to do some experimentation to find out what works for your models and dataset. - -@anchor background -@par Technical Background - -For an introduction to deep learning foundations, including MLP and CNN, -refer to [7]. - -This module trains many models a time across the database cluster in order -to explore network architectures and hyperparameters. It uses model hopper -parallelism (MOP) and has high convergence efficiency since it does not do -model averaging [2]. - -On the effect of database cluster size: as the database cluster size increases, -it will be faster to train a set of models, as long as you have at -least as many model selection tuples as segments. This is because model state is "hopped" from -segment to segment and training takes place in parallel [1,2]. If you have fewer model -selection tuples to train than segments, then some -segments may not be busy 100% of the time so speedup will not necessarily increase -on a larger cluster. Inference (predict) is an embarrassingly parallel operation so -inference runtimes will be proportionally faster as the number of segments increases. - -@anchor literature -@literature - -@anchor mlp-lit-1 -[1] "Cerebro: Efficient and Reproducible Model Selection on Deep Learning Systems," -Supun Nakandala, Yuhao Zhang, and Arun Kumar, ACM SIGMOD 2019 DEEM Workshop, -https://adalabucsd.github.io/papers/2019_Cerebro_DEEM.pdf - -[2] "Resource-Efficient and Reproducible Model Selection on Deep Learning Systems," -Supun Nakandala, Yuhao Zhang, and Arun Kumar, Technical Report, Computer Science and -Engineering, University of California, San Diego -https://adalabucsd.github.io/papers/TR_2019_Cerebro.pdf - -[3] https://keras.io/ - -[4] https://www.tensorflow.org/ - -[5] "Neural Networks for Machine Learning", Lectures 6a and 6b on mini-batch gradient descent, -Geoffrey Hinton with Nitish Srivastava and Kevin Swersky, -http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf - -[6] Deep learning section of Apache MADlib wiki https://cwiki.apache.org/confluence/display/MADLIB/Deep+Learning - -[7] Deep Learning, Ian Goodfellow, Yoshua Bengio and Aaron Courville, MIT Press, 2016. - -[8] Greenplum Database server configuration parameters https://gpdb.docs.pivotal.io/latest/ref_guide/config_params/guc-list.html - -@anchor related -@par Related Topics - -File madlib_keras_fit_multiple_model.sql_in documents training, evaluate and predict functions. - -*/ - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model( - source_table VARCHAR, - model_output_table VARCHAR, - model_selection_table VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR, - metrics_compute_frequency INTEGER, - warm_start BOOLEAN, - name VARCHAR, - description VARCHAR -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras_fit_multiple_model') - from utilities.control import SetGUC - with AOControl(False): - with SetGUC("plan_cache_mode", "force_generic_plan"): - fit_obj = madlib_keras_fit_multiple_model.FitMultipleModel(**globals()) -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model( - source_table VARCHAR, - model_output_table VARCHAR, - model_selection_table VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR, - metrics_compute_frequency INTEGER, - warm_start BOOLEAN, - name VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model( - source_table VARCHAR, - model_output_table VARCHAR, - model_selection_table VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR, - metrics_compute_frequency INTEGER, - warm_start BOOLEAN -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, $6, $7, $8, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model( - source_table VARCHAR, - model_output_table VARCHAR, - model_selection_table VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR, - metrics_compute_frequency INTEGER -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, $6, $7, FALSE, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model( - source_table VARCHAR, - model_output_table VARCHAR, - model_selection_table VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN, - validation_table VARCHAR -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, $6, NULL, FALSE, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_fit_multiple_model( - source_table VARCHAR, - model_output_table VARCHAR, - model_selection_table VARCHAR, - num_iterations INTEGER, - use_gpus BOOLEAN -) RETURNS VOID AS $$ - SELECT MADLIB_SCHEMA.madlib_keras_fit_multiple_model($1, $2, $3, $4, $5, NULL, NULL, FALSE, NULL, NULL); -$$ LANGUAGE sql VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fit_transition_multiple_model( - state BYTEA, - dependent_var BYTEA, - independent_var BYTEA, - dependent_var_shape INTEGER[], - independent_var_shape INTEGER[], - model_architecture TEXT, - compile_params TEXT, - fit_params TEXT, - dist_key INTEGER, - dist_key_mapping INTEGER[], - current_seg_id INTEGER, - segments_per_host INTEGER, - images_per_seg INTEGER[], - use_gpus BOOLEAN, - accessible_gpus_for_seg INTEGER[], - prev_serialized_weights BYTEA, - is_final_iteration BOOLEAN -) RETURNS BYTEA AS $$ -PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras') - return madlib_keras.fit_transition(is_multiple_model = True, **globals()) -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.fit_step_multiple_model( - BYTEA, - BYTEA, - INTEGER[], - INTEGER[], - TEXT, - TEXT, - TEXT, - INTEGER, - INTEGER[], - INTEGER, - INTEGER, - INTEGER[], - BOOLEAN, - INTEGER[], - BYTEA, - BOOLEAN); -CREATE AGGREGATE MADLIB_SCHEMA.fit_step_multiple_model( - /* dependent_var */ BYTEA, - /* independent_var */ BYTEA, - /* dependent_var_shape */ INTEGER[], - /* independent_var_shape */ INTEGER[], - /* model_architecture */ TEXT, - /* compile_params */ TEXT, - /* fit_params */ TEXT, - /* dist_key */ INTEGER, - /* dist_key_mapping */ INTEGER[], - /* current_seg_id */ INTEGER, - /* segments_per_host */ INTEGER, - /* images_per_seg */ INTEGER[], - /* use_gpus */ BOOLEAN, - /* accessible_gpus_for_seg */ INTEGER[], - /* prev_serialized_weights */ BYTEA, - /* is_final_iteration */ BOOLEAN -)( - STYPE=BYTEA, - SFUNC=MADLIB_SCHEMA.fit_transition_multiple_model -); diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.py_in deleted file mode 100644 index 4eec8a7..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.py_in +++ /dev/null @@ -1,256 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -@file madlib_keras_gpu_info.py_in - -@brief GPU configuration helper function - -@namespace madlib_keras_gpu_info -""" - -import os -import subprocess - -import plpy -from utilities.utilities import is_platform_pg -from utilities.utilities import unique_string -from utilities.validate_args import output_tbl_valid - -class OutputInfoSchema: - TEMP_INFO_TABLE = unique_string(desp='gpu_info') - SEG_ID_COL = 'gp_seg_id' - GPU_DESCR_COL = 'gpu_descr' - - -class Source: - NVIDIA = 'nvidia' - TENSORFLOW = 'tensorflow' - - -class GPUInfoFunctions: - @staticmethod - def get_gpu_info_from_nvidia(): - """ - This function will run only on segment(s). Make sure not to run any non - select plpy execute. - :return: list of gpu descriptions as returned by nvidia-smi -L. - """ - try: - return subprocess.check_output(["nvidia-smi", "-L"]).splitlines() - except Exception: # Handle case when nvidia-smi -L fails - return [] - - @staticmethod - def get_gpu_info_from_tensorflow(): - """ - This function will run only on segment(s). Make sure not to run any non - select plpy execute. - :return: list of gpu descriptions as returned by tensorflow - """ - current_working_dir = os.path.dirname(os.path.realpath(__file__)) - gpus = subprocess.check_output(["python", "gpu_info_from_tf.py"], - cwd=current_working_dir).splitlines() - return gpus - - -def gpu_configuration(schema_madlib, output_table, source): - """ - :return: List of gpus along with their hostname in the format - GPDB - gpu_descr | hostname - ------------------+-------------------------- - NVIDIA Tesla P100 | host1 - NVIDIA Tesla P100 | host1 - Super Duper GPU | host2 - Super Duper GPU | host2 - 1. We use gp_dist_random to run either the nvidia smi UDF or the tensorflow UDF - on all the hosts. (see gpu_info_nvidia/gpu_info_tensorflow) - 2. Also we do not need to run the tf/nvidia UDF on all the segments, - just one segment per host. That's why we group the output of - gp_segment_configuration by hostname and get the min segment from - each host. - 3. To get the hostname along with the gpu description, we have to join - the output of nvidia/tf UDF with gp_segment_configuration and filter - out the following - * master - * mirror segments - * empty/null gpu description - Workflow for gpdb - 1. Run query to get min seg ids on each host. This is so that we can run - the gpu UDF on just one segment per host. - 2. Create a table by running the tf/nvidia UDF on the segment ids returned - from the previous step. Note that this table will only contain the output - of the UDF and the segment id itself. This table does not contain hostnames - 3. To get the hostname associated with the segment id, we need to join the - table created in step with gp_segment_configuration. - It's important to note that we can merge all these 3 queries into one but - the problem with that is that a redistribution happens before running the UDF - which means the UDF does not run on the segments that we pass in to the query. - To avoid this, we broke down the query into 3 parts so that the UDF is always - run on the intended segments. - - POSTGRES - gpu_descr | hostname - ------------------+-------------------------- - NVIDIA Tesla P100 | localhost - """ - module_name = 'madlib_keras_gpu_info' - output_tbl_valid(output_table, module_name) - - if not source: - source = Source.TENSORFLOW - source = source.lower() - if source != Source.TENSORFLOW and source != Source.NVIDIA: - plpy.error("DL: source has to be one of {0} or {1}".format( - Source.TENSORFLOW, Source.NVIDIA)) - - gpu_fn_name = 'gpu_info_{0}'.format(source) - if is_platform_pg(): - gpu_for_postgres(schema_madlib, output_table, gpu_fn_name) - else: - gpu_for_gpdb(schema_madlib, output_table, gpu_fn_name) - - -def gpu_for_postgres(schema_madlib, output_table, gpu_fn_name): - gpu_info_query = """ - CREATE TABLE {0} AS - SELECT 'localhost' as hostname, {1} from (SELECT unnest({2}.{3}()) AS {1}) s1 - where {1} is NOT NULL AND {1} != '' - """.format(output_table, OutputInfoSchema.GPU_DESCR_COL, - schema_madlib, gpu_fn_name) - plpy.execute(gpu_info_query) - - -def gpu_for_gpdb(schema_madlib, output_table, gpu_fn_name): - min_seg_on_each_host = get_min_seg_ids_on_each_host() - - create_gpu_info_table_without_hostname(schema_madlib, gpu_fn_name, - min_seg_on_each_host) - - create_gpu_info_table_with_hostname(output_table) - - plpy.execute("DROP TABLE IF EXISTS {0}".format(OutputInfoSchema.TEMP_INFO_TABLE)) - - -def get_min_seg_ids_on_each_host(): - """ - Run query to get min seg ids on each host. This is so that we can run - the gpu UDF on just one segment per host. - :return: List of min seg id per host - """ - min_seg_id_alias = 'min_seg_id' - min_seg_query = """ - SELECT {min_seg_id_alias} FROM - (select hostname, min(content) AS {min_seg_id_alias} - FROM gp_segment_configuration WHERE content != -1 AND role='p' - GROUP BY hostname) min_seg_id_subquery - """.format(**locals()) - min_seg_on_each_host = plpy.execute(min_seg_query) - - min_seg_on_each_host = ','.join([str(seg[min_seg_id_alias]) - for seg in min_seg_on_each_host]) - return min_seg_on_each_host - - -def create_gpu_info_table_without_hostname(schema_madlib, gpu_fn_name, - min_seg_on_each_host): - """ output_table, - Create a table by running the tf/nvidia UDF on the segment ids returned - from the previous step. Note that this table will only contain the output - of the UDF and the segment id itself. This table does not contain hostnames - :param schema_madlib: - :param gpu_fn_name: - :param min_seg_on_each_host: - """ - gpu_info_per_host_query = """ - CREATE TABLE {0} AS SELECT gp_segment_id AS {1}, {2}.{3}() - AS {4} FROM gp_dist_random('gp_id') WHERE gp_segment_id IN ({5}) - """.format(OutputInfoSchema.TEMP_INFO_TABLE, - OutputInfoSchema.SEG_ID_COL, - schema_madlib, gpu_fn_name, - OutputInfoSchema.GPU_DESCR_COL, - min_seg_on_each_host) - plpy.execute(gpu_info_per_host_query) - - -def create_gpu_info_table_with_hostname(output_table): - """ - Create the final output table that contains the hostname and the gpu description. - To create this table, we need to join the table created in - create_gpu_info_table_without_hostname with gp_segment_configuration. - """ - final_join_query = """ - CREATE TABLE {0} AS - SELECT hostname, {1} FROM - ( - SELECT hostname, unnest({1}) AS {1} FROM {2} - JOIN - gp_segment_configuration ON {3}=content WHERE content != -1 AND role='p' - ) s1 - WHERE {1} != '' AND {1} is NOT NULL ORDER BY 1,2; - """.format(output_table, OutputInfoSchema.GPU_DESCR_COL, - OutputInfoSchema.TEMP_INFO_TABLE, - OutputInfoSchema.SEG_ID_COL) - plpy.execute(final_join_query) - - -def gpu_configuration_help(schema_madlib): - """ - Help function for gpu configuration - - Args: - @param schema_madlib - - Returns: - String. Help/usage information - """ - - help_string = """ - Utility function to report number and type of GPUs on the database cluster. - ----------------------------------------------------------------------- - USAGE - ----------------------------------------------------------------------- - SELECT {schema_madlib}.gpu_configuration( - output_table, -- Name of the output table to write out the - GPU information. - source -- Default: 'tensorflow'. Source for determining - GPU configuration. - Using 'tensorflow' returns a description based - on what TensorFlow reports. - Using 'nvidia' returns a description based - on what the Nvidia Systems Management Interface - (nvidia-smi) reports [1]. - Note that MADlib and Keras will use the TensorFlow - information; the lower level nvidia-smi info - is provided for convenience. - ) - ); - - ----------------------------------------------------------------------- - OUTPUT - ----------------------------------------------------------------------- - The output table ('output_table' above) contains the following columns: - - hostname: Name of the host machine in the cluster. - Does not include master or mirrors. For PostgreSQL this will - always return 'localhost'. - gpu_descr: String reported by TensorFlow or nvidia-smi. - """ - - return help_string.format(schema_madlib=schema_madlib) diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.sql_in deleted file mode 100644 index d2418e4..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_gpu_info.sql_in +++ /dev/null @@ -1,279 +0,0 @@ -/* ----------------------------------------------------------------------- *//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * - * @file madlib_keras_gpu_info.sql_in - * - * @brief SQL functions for GPU configuration. - * @date Nov 2019 - * - * - *//* ----------------------------------------------------------------------- */ - -m4_include(`SQLCommon.m4') - -/** -@addtogroup grp_gpu_configuration - -@brief Utility function to report number and type of GPUs in the database cluster. - -\warning This MADlib method is still in early stage development. -Interface and implementation are subject to change. - - - -This utility function reports the number and type of GPUs -attached to hosts on the database cluster. -This can be useful when determining which segments to use for -training deep neural nets. For example, for economic reasons -you may wish to set up a heterogeneous clusters with GPUs only -on some of the hosts, not all of them. This utility -can help you identify where the GPUS are and direct the compute -to those locations only in subsequent training and evaluation steps. - -@anchor get_gpu_config -@par GPU Confuguration - -
-gpu_configuration( output_table,
-                   source
-	               )
-
-\b Arguments -
-
output_table
-
TEXT. Name of the output table to write out the GPU information. -
- -
source (optional)
-
TEXT, default: 'tensorflow'. Source for determining GPU configuration. - Using 'tensorflow' returns a description based on what TensorFlow reports. - Using 'nvidia' returns a description based on what the Nvidia Systems - Management Interface (nvidia-smi) reports [1]. Note that MADlib and Keras will use the - TensorFlow information; the lower level nvidia-smi info is provided for convenience. - - @note - If the command 'nvidia-smi -L' returns an error, we do not pass through - the error message, but instead will show no GPUs for that host in the output table. - You may want to run nvidia-smi from the command line to see error - and informational messages. - -
-
- -Output -
- The output table contains the following: -
idGives the 'id' for each prediction, corresponding to each row from the test_table.
estimated_COL_NAME - (For pred_type='response') The estimated class - for classification, where - COL_NAME is the name of the column to be - predicted from test data. -
prob_CLASS - (For pred_type='prob' for classification) The - probability of a given class. - There will be one column for each class - in the training data. -
- - - - - - - - -
hostnameTEXT. Name of the host machine in the cluster. - Does not include master or mirrors. For PostgreSQL - this will always return 'localhost'. -
gpu_descrTEXT. String reported by TensorFlow or nvidia-smi. - The formats are different and shown in the examples below. -
-
- -@anchor example -@par Examples - --# Get GPU configuration as per TensorFlow: -
-DROP TABLE IF EXISTS host_gpu_mapping_tf;
-SELECT * FROM madlib.gpu_configuration('host_gpu_mapping_tf');
-SELECT * FROM host_gpu_mapping_tf ORDER BY hostname, gpu_descr;
-
-
- hostname |                                        gpu_descr
-----------+------------------------------------------------------------------------------------------
- phoenix0 | device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0
- phoenix0 | device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0
- phoenix0 | device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0
- phoenix0 | device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0
- phoenix1 | device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0
- phoenix1 | device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0
- phoenix3 | device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0
- phoenix3 | device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0
- phoenix3 | device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0
- phoenix3 | device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0
- phoenix4 | device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0
- phoenix4 | device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:05.0, compute capability: 6.0
- phoenix4 | device: 2, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:06.0, compute capability: 6.0
- phoenix4 | device: 3, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0
-(14 rows)
-
-In this heterogeneous cluster there are 4 GPUs attached to hosts 0, 3 and 4. There are 2 GPUs -attached to host 1 and no GPUs attached to host 2. - --# Get GPU configuration as per nvidia-smi: -
-DROP TABLE IF EXISTS host_gpu_mapping_nvidia;
-SELECT * FROM madlib.gpu_configuration('host_gpu_mapping_nvidia', -- output table
-                                       'nvidia'                   -- source for GPU info
-                                       );
-SELECT * FROM host_gpu_mapping_nvidia ORDER BY hostname, gpu_descr;
-
-
- hostname |                                  gpu_descr
-----------+------------------------------------------------------------------------------
- phoenix0 | GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-f2ccc77e-2501-f6ee-4754-069dda256fb2)
- phoenix0 | GPU 1: Tesla P100-PCIE-16GB (UUID: GPU-b1fc40ca-c7c6-bc86-f20f-6e9a62cda3f8)
- phoenix0 | GPU 2: Tesla P100-PCIE-16GB (UUID: GPU-d93bb21b-96f9-7c1d-3bab-cdd92b7bbc9d)
- phoenix0 | GPU 3: Tesla P100-PCIE-16GB (UUID: GPU-2d79c4a8-479e-2f33-39f8-3ba80b63f830)
- phoenix1 | GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-0af6bb1e-5b5b-4988-ad3a-a917e9584702)
- phoenix1 | GPU 1: Tesla P100-PCIE-16GB (UUID: GPU-d824c976-a8aa-ef26-a13c-9a9a7fe86bfd)
- phoenix3 | GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-3681d0b6-1ec6-0453-fd81-29d88e549cd9)
- phoenix3 | GPU 1: Tesla P100-PCIE-16GB (UUID: GPU-d4b1f2e7-b238-ac9a-bbfe-918adeb69472)
- phoenix3 | GPU 2: Tesla P100-PCIE-16GB (UUID: GPU-42a32ef1-a60c-e599-c8cf-0e669111ab6f)
- phoenix3 | GPU 3: Tesla P100-PCIE-16GB (UUID: GPU-1cce09c4-6856-8031-be0b-8e8bbf9a10f3)
- phoenix4 | GPU 0: Tesla P100-PCIE-16GB (UUID: GPU-a71bdc18-fdd5-ba25-617e-19b23cc8e827)
- phoenix4 | GPU 1: Tesla P100-PCIE-16GB (UUID: GPU-f9d13688-7fe6-a029-24d1-985a5659f18f)
- phoenix4 | GPU 2: Tesla P100-PCIE-16GB (UUID: GPU-06a7f54b-c07a-e87a-20d6-09bd99b19531)
- phoenix4 | GPU 3: Tesla P100-PCIE-16GB (UUID: GPU-af3b32f3-8bd9-cb75-a8fb-25253b9da926)
-(14 rows)
-
- --# To get a fuller picture at the segment level, combine with the Greenplum catalog -table 'gp_segment_configuration' which contains information about -segment instance configuration [2]. Here is an example of this table -filtering out master and mirrors: -
-SELECT * FROM gp_segment_configuration WHERE role='p' AND content>=0 ORDER BY hostname, dbid;
-
-
- dbid | content | role | preferred_role | mode | status | port  | hostname | address  | replication_port
-------+---------+------+----------------+------+--------+-------+----------+----------+------------------
-    2 |       0 | p    | p              | c    | u      | 40000 | phoenix0 | phoenix0 |            70000
-    3 |       1 | p    | p              | c    | u      | 40001 | phoenix0 | phoenix0 |            70001
-    4 |       2 | p    | p              | c    | u      | 40002 | phoenix0 | phoenix0 |            70002
-    5 |       3 | p    | p              | c    | u      | 40003 | phoenix0 | phoenix0 |            70003
-    6 |       4 | p    | p              | c    | u      | 40000 | phoenix1 | phoenix1 |            70000
-    7 |       5 | p    | p              | c    | u      | 40001 | phoenix1 | phoenix1 |            70001
-    8 |       6 | p    | p              | c    | u      | 40002 | phoenix1 | phoenix1 |            70002
-    9 |       7 | p    | p              | c    | u      | 40003 | phoenix1 | phoenix1 |            70003
-   10 |       8 | p    | p              | c    | u      | 40000 | phoenix2 | phoenix2 |            70000
-   11 |       9 | p    | p              | c    | u      | 40001 | phoenix2 | phoenix2 |            70001
-   12 |      10 | p    | p              | c    | u      | 40002 | phoenix2 | phoenix2 |            70002
-   13 |      11 | p    | p              | c    | u      | 40003 | phoenix2 | phoenix2 |            70003
-   14 |      12 | p    | p              | c    | u      | 40000 | phoenix3 | phoenix3 |            70000
-   15 |      13 | p    | p              | c    | u      | 40001 | phoenix3 | phoenix3 |            70001
-   16 |      14 | p    | p              | c    | u      | 40002 | phoenix3 | phoenix3 |            70002
-   17 |      15 | p    | p              | c    | u      | 40003 | phoenix3 | phoenix3 |            70003
-   18 |      16 | p    | p              | c    | u      | 40000 | phoenix4 | phoenix4 |            70000
-   19 |      17 | p    | p              | c    | u      | 40001 | phoenix4 | phoenix4 |            70001
-   20 |      18 | p    | p              | c    | u      | 40002 | phoenix4 | phoenix4 |            70002
-   21 |      19 | p    | p              | c    | u      | 40003 | phoenix4 | phoenix4 |            70003
-(20 rows)
-
-Now join this table with the GPU resources table to create a table containing a -list of all segments on hosts with GPUs attached: -
-DROP TABLE IF EXISTS segments_to_use;
-CREATE TABLE segments_to_use AS
-  SELECT DISTINCT dbid, hostname FROM gp_segment_configuration JOIN host_gpu_mapping_tf USING (hostname)
-  WHERE role='p' AND content>=0;
-SELECT * FROM segments_to_use ORDER BY hostname, dbid;
-
-
- dbid | hostname
-------+----------
-    2 | phoenix0
-    3 | phoenix0
-    4 | phoenix0
-    5 | phoenix0
-    6 | phoenix1
-    7 | phoenix1
-    8 | phoenix1
-    9 | phoenix1
-   14 | phoenix3
-   15 | phoenix3
-   16 | phoenix3
-   17 | phoenix3
-   18 | phoenix4
-   19 | phoenix4
-   20 | phoenix4
-   21 | phoenix4
-(16 rows)
-
- -@anchor references -@par References - -[1] Nvidia Systems Management Interface (nvidia-smi) https://developer.nvidia.com/nvidia-system-management-interface - -[2] Greenplum 'gp_segment_configuration' table https://gpdb.docs.pivotal.io/latest/ref_guide/system_catalogs/gp_segment_configuration.html - -@anchor related -@par Related Topics - -See madlib_keras_gpu_info.sql_in - -*/ - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_info_tensorflow() RETURNS TEXT[] as -$$ - PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras_gpu_info') - return madlib_keras_gpu_info.GPUInfoFunctions.get_gpu_info_from_tensorflow() -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_info_nvidia() RETURNS TEXT[] as -$$ - PythonFunctionBodyOnlyNoSchema(`deep_learning', `madlib_keras_gpu_info') - return madlib_keras_gpu_info.GPUInfoFunctions.get_gpu_info_from_nvidia() -$$ LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_configuration(output_table text, source text) -RETURNS VOID AS -$$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras_gpu_info') - from utilities.control import MinWarning - with AOControl(False) and MinWarning("error"): - madlib_keras_gpu_info.gpu_configuration(schema_madlib, output_table, source) -$$ -LANGUAGE plpythonu -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_configuration(output_table text) -RETURNS VOID AS -$$ - SELECT MADLIB_SCHEMA.gpu_configuration($1, NULL); -$$ -LANGUAGE sql; - - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gpu_configuration() - RETURNS VARCHAR AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras_gpu_info') - return madlib_keras_gpu_info.gpu_configuration_help(schema_madlib) -$$ LANGUAGE plpythonu IMMUTABLE - m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in deleted file mode 100644 index 5be078b..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in +++ /dev/null @@ -1,316 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import numpy as np -from model_arch_info import ModelArchSchema -from utilities.utilities import add_postfix -from utilities.utilities import unique_string -from utilities.utilities import is_platform_pg -from utilities.validate_args import table_exists -from madlib_keras_gpu_info import GPUInfoFunctions -import plpy - - -############### Constants used in other deep learning files ######### -# Name of columns in model summary table. -CLASS_VALUES_COLNAME = "class_values" -NORMALIZING_CONST_COLNAME = "normalizing_const" -COMPILE_PARAMS_COLNAME = "compile_params" -DEPENDENT_VARNAME_COLNAME = "dependent_varname" -DEPENDENT_VARTYPE_COLNAME = "dependent_vartype" -INDEPENDENT_VARNAME_COLNAME = "independent_varname" -MODEL_ARCH_TABLE_COLNAME = "model_arch_table" -MODEL_ID_COLNAME = ModelArchSchema.MODEL_ID -MODEL_WEIGHTS_COLNAME = "model_weights" -METRIC_TYPE_COLNAME = "metrics_type" - -# Name of independent, dependent and distribution key colnames in batched table. -# These are readonly variables, do not modify. -# MADLIB-1300 Adding these variables for DL only at this time. -MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL = "dependent_var" -MINIBATCH_OUTPUT_INDEPENDENT_COLNAME_DL = "independent_var" -DISTRIBUTION_KEY_COLNAME = "__dist_key__" -## sql variable types -FLOAT32_SQL_TYPE = 'REAL' -SMALLINT_SQL_TYPE = 'SMALLINT' - -DEFAULT_NORMALIZING_CONST = 1.0 -GP_SEGMENT_ID_COLNAME = "gp_segment_id" -INTERNAL_GPU_CONFIG = '__internal_gpu_config__' - -##################################################################### - -# Prepend a dimension to np arrays using expand_dims. -def expand_input_dims(input_data): - input_data = np.array(input_data, dtype=np.float32) - input_data = np.expand_dims(input_data, axis=0) - return input_data - -def np_array_float32(var, var_shape): - arr = np.frombuffer(var, dtype=np.float32) - arr.shape = var_shape - return arr - -def np_array_int16(var, var_shape): - arr = np.frombuffer(var, dtype=np.int16) - arr.shape = var_shape - return arr - -def strip_trailing_nulls_from_class_values(class_values): - """ - class_values is a list of unique class levels in training data. This - could have multiple Nones in it, and this function strips out all the - Nones that occur after the first element in the list. - Examples: - 1) input class_values = ['cat', 'dog'] - output class_values = ['cat', 'dog'] - - 2) input class_values = [None, 'cat', 'dog'] - output class_values = [None, 'cat', 'dog'] - - 3) input class_values = [None, 'cat', 'dog', None, None] - output class_values = [None, 'cat', 'dog'] - - 4) input class_values = ['cat', 'dog', None, None] - output class_values = ['cat', 'dog'] - - 5) input class_values = [None, None] - output class_values = [None] - @args: - @param: class_values, list - @returns: - updated class_values list - """ - num_of_valid_class_values = 0 - if class_values is not None: - for ele in class_values: - if ele is None and num_of_valid_class_values > 0: - break - num_of_valid_class_values += 1 - # Pass only the valid class_values for creating columns - class_values = class_values[:num_of_valid_class_values] - return class_values - -def get_image_count_per_seg_from_array(current_seg_id, images_per_seg): - """ - Get the image count from the array containing all the images - per segment. Based on the platform, we find the index of the current segment. - This function is only called from inside the transition function. - """ - if is_platform_pg(): - total_images = images_per_seg[0] - else: - total_images = images_per_seg[current_seg_id] - return total_images - -def get_image_count_per_seg_for_minibatched_data_from_db(table_name): - """ - Query the given minibatch formatted table and return the total rows per segment. - Since we cannot pass a dictionary to the keras fit step function we create - arrays out of the segment numbers and the rows per segment values. - This function assumes that the table is not empty and is minibatched which means - that it would have been distributed by __dist_key__. - :param table_name: - :return: Returns two arrays - 1. An array containing all the segment numbers in ascending order - 1. An array containing the total images on each of the segments in the - segment array. - """ - - mb_dep_var_col = MINIBATCH_OUTPUT_DEPENDENT_COLNAME_DL - - shape_col = add_postfix(mb_dep_var_col, "_shape") - - if is_platform_pg(): - res = plpy.execute( - """ SELECT {0}::SMALLINT[] AS shape - FROM {1} - """.format(shape_col, table_name)) - images_per_seg = [sum(r['shape'][0] for r in res)] - seg_ids = [0] - else: - # The number of images in the buffer is the first dimension in the shape. - # Using __dist_key__ instead of gp_segment_id: Since gp_segment_id is - # not the actual distribution key column, the optimizer/planner - # generates a plan with Redistribute Motion, creating multiple slices on - # each segment. For DL, since GPU memory allocation is tied to the process - # where it is initialized, we want to minimize creating any additional - # slices per segment. This is mainly to avoid any GPU memory allocation - # failures which can occur when a newly created slice(process) tries - # allocating GPU memory which is already allocated by a previously - # created slice(process). - # Since the minibatch_preprocessor evenly distributes the data with __dist_key__ - # as the input table's distribution key, using this for calculating - # total images on each segment will avoid creating unnecessary slices(processes). - images_per_seg = plpy.execute( - """ SELECT {0}, sum({1}[1]) AS images_per_seg - FROM {2} - GROUP BY {0} - """.format(DISTRIBUTION_KEY_COLNAME, shape_col, table_name)) - seg_ids = [int(each_segment[DISTRIBUTION_KEY_COLNAME]) - for each_segment in images_per_seg] - images_per_seg = [int(each_segment["images_per_seg"]) - for each_segment in images_per_seg] - - return seg_ids, images_per_seg - -def get_image_count_per_seg_for_non_minibatched_data_from_db(table_name): - """ - Query the given non minibatch formatted table and return the total rows per segment. - Since we cannot pass a dictionary to the keras fit step function we create arrays - out of the segment numbers and the rows per segment values. - This function assumes that the table is not empty. - :param table_name: - :return: gp segment id col name and two arrays - 1. An array containing all the segment numbers in ascending order - 2. An array containing the total rows for each of the segments in the - segment array - """ - if is_platform_pg(): - images_per_seg = plpy.execute( - """ SELECT count(*) AS images_per_seg - FROM {0} - """.format(table_name)) - seg_ids = [0] - gp_segment_id_col = '0' - else: - # Compute total buffers on each segment - images_per_seg = plpy.execute( - """ SELECT {0}, count(*) AS images_per_seg - FROM {1} - GROUP BY {0} - """.format(GP_SEGMENT_ID_COLNAME, table_name)) - seg_ids = [int(image[GP_SEGMENT_ID_COLNAME]) for image in images_per_seg] - gp_segment_id_col = '{0}.{1}'.format(table_name,GP_SEGMENT_ID_COLNAME) - - images_per_seg = [int(image["images_per_seg"]) for image in images_per_seg] - return gp_segment_id_col, seg_ids, images_per_seg - -def parse_shape(shape): - # Parse the shape format given by the sql into an int array - # [1:10][1:32][1:3] -> [10, 32, 3] - # Split on :, discard the first one [1:], - # split each piece on ], take the first piece [0], convert to int - return [int(a.split(']')[0]) for a in shape.split(':')[1:]] - - -def query_model_configs(model_selection_table, model_selection_summary_table, - mst_key_col, model_arch_table_col): - msts_query = """ - SELECT * FROM {model_selection_table} - ORDER BY {mst_key_col} - """.format(**locals()) - model_arch_table_query = """ - SELECT {model_arch_table_col} - FROM {model_selection_summary_table} - """.format(**locals()) - msts = list(plpy.execute(msts_query)) - model_arch_table = plpy.execute(model_arch_table_query)[0][model_arch_table_col] - return msts, model_arch_table - -def query_dist_keys(source_table, dist_key_col): - """ Read distinct keys from the source table """ - dist_key_query = """ - SELECT DISTINCT({dist_key_col}) FROM {source_table} - ORDER BY {dist_key_col} - """.format(dist_key_col=dist_key_col, - source_table=source_table) - res = list(plpy.execute(dist_key_query)) - res = [x[dist_key_col] for x in res] - return res - -def query_weights(model_output_table, model_weights_col, mst_key_col, mst_key): - mlp_weights_query = """ - SELECT {model_weights_col}, {mst_key_col} - FROM {model_output_table} - WHERE {mst_key_col} = {mst_key} - """.format(**locals()) - - res = plpy.execute(mlp_weights_query) - return res[0][model_weights_col] - -def create_summary_view(module_name, model_table, mst_key): - tmp_view_summary = unique_string('tmp_view_summary') - model_summary_table = add_postfix(model_table, "_summary") - model_info_table = add_postfix(model_table, "_info") - if not (table_exists(model_summary_table) and - table_exists(model_info_table)): - plpy.error("{0}: Missing summary and/or info tables for {1}".format( - module_name, model_table)) - - res = plpy.execute(""" - SELECT mst_key FROM {model_info_table} WHERE mst_key = {mst_key} - """.format(**locals())) - if len(res) < 1: - plpy.error("{0}: mst_key {1} does not exist in the info table".format( - module_name, mst_key)) - - plpy.execute(""" - CREATE VIEW {tmp_view_summary} AS - SELECT * - FROM {model_summary_table}, {model_info_table} - WHERE mst_key = {mst_key} - """.format(**locals())) - return tmp_view_summary - - -def get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name): - if is_platform_pg(): - gpus = GPUInfoFunctions.get_gpu_info_from_tensorflow() - if not gpus: - plpy.error("{0} error: No GPUs configured on host.".format(module_name)) - return [len(gpus)] - else: - gpu_info_table = unique_string(desp = 'gpu_info') - gpu_table_query = """ - SELECT {schema_madlib}.gpu_configuration('{gpu_info_table}') - """.format(**locals()) - plpy.execute(gpu_table_query) - gpu_query = """ - SELECT hostname, count(*) AS count FROM {gpu_info_table} GROUP BY hostname - """.format(**locals()) - gpu_query_result = plpy.execute(gpu_query) - plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table)) - if not gpu_query_result: - plpy.error("{0} error: No GPUs configured on hosts.".format(module_name)) - - host_dict = {} - for i in gpu_query_result: - host_dict[i['hostname']] = int(i['count']) - - seg_query = """ - SELECT hostname, content AS segment_id - FROM gp_segment_configuration - WHERE content != -1 AND role = 'p' - """ - seg_query_result = plpy.execute(seg_query) - - accessible_gpus_for_seg = [0] * len(seg_query_result) - warning_flag = True - for i in seg_query_result: - if i['hostname'] in host_dict.keys(): - accessible_gpus_for_seg[i['segment_id']] = host_dict[i['hostname']] - if 0 < accessible_gpus_for_seg[i['segment_id']] < segments_per_host and warning_flag: - plpy.warning( - 'The number of GPUs per segment host is less than the number of ' - 'segments per segment host. When different segments share the ' - 'same GPU, this may fail in some scenarios. The current ' - 'recommended configuration is to have 1 GPU available per segment.') - warning_flag = False - return accessible_gpus_for_seg diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in deleted file mode 100644 index f3b02c7..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in +++ /dev/null @@ -1,192 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations - -import plpy -from collections import OrderedDict -from madlib_keras_validator import MstLoaderInputValidator -from utilities.control import MinWarning -from utilities.utilities import add_postfix -from madlib_keras_wrapper import convert_string_of_args_to_dict -from keras_model_arch_table import ModelArchSchema - -class ModelSelectionSchema: - MST_KEY = 'mst_key' - MODEL_ID = ModelArchSchema.MODEL_ID - MODEL_ARCH_TABLE = 'model_arch_table' - COMPILE_PARAMS = 'compile_params' - FIT_PARAMS = 'fit_params' - col_types = ('SERIAL', 'INTEGER', 'VARCHAR', 'VARCHAR') - -@MinWarning("warning") -class MstLoader(): - """The utility class for loading a model selection table with model parameters. - - Currently just takes all combinations of input parameters passed. This - utility validates the inputs. - - Attributes: - compile_params_list (list): The input list of compile params choices. - fit_params_list (list): The input list of fit params choices. - model_id_list (list): The input list of model id choices. - model_arch_table (str): The name of model architechure table. - model_selection_table (str): The name of the output mst table. - msts (list): The list of generated msts. - - """ - - def __init__(self, - model_arch_table, - model_selection_table, - model_id_list, - compile_params_list, - fit_params_list, - **kwargs): - - self.model_arch_table = model_arch_table - self.model_selection_table = model_selection_table - self.model_selection_summary_table = add_postfix( - model_selection_table, "_summary") - self.model_id_list = sorted(list(set(model_id_list))) - MstLoaderInputValidator( - model_arch_table=self.model_arch_table, - model_selection_table=self.model_selection_table, - model_selection_summary_table=self.model_selection_summary_table, - model_id_list=self.model_id_list, - compile_params_list=compile_params_list, - fit_params_list=fit_params_list - ) - self.compile_params_list = self.params_preprocessed( - compile_params_list) - self.fit_params_list = self.params_preprocessed(fit_params_list) - - self.msts = [] - - self.find_combinations() - - def load(self): - """The entry point for loading the model selection table. - """ - # All of the side effects happen in this function. - self.create_mst_table() - self.create_mst_summary_table() - self.insert_into_mst_table() - - def params_preprocessed(self, list_strs): - """Preprocess the input lists. Eliminate white spaces and sort them. - - Args: - list_strs (list): A list of strings. - - Returns: - list: The preprocessed list of strings. - """ - - dict_dedup = {} - for string in list_strs: - d = convert_string_of_args_to_dict(string) - hash_tuple = tuple( '{0} = {1}'\ - .format(x, d[x]) for x in sorted(d.keys())) - dict_dedup[hash_tuple] = string - - return dict_dedup.values() - - def find_combinations(self): - """Backtracking helper for generating the combinations. - """ - param_grid = OrderedDict([ - (ModelSelectionSchema.MODEL_ID, self.model_id_list), - (ModelSelectionSchema.COMPILE_PARAMS, self.compile_params_list), - (ModelSelectionSchema.FIT_PARAMS, self.fit_params_list) - ]) - - def find_combinations_helper(msts, p, i): - param_names = param_grid.keys() - if i < len(param_names): - for x in param_grid[param_names[i]]: - p[param_names[i]] = x - find_combinations_helper(msts, p, i + 1) - else: - msts.append(p.copy()) - find_combinations_helper(self.msts, {}, 0) - - def create_mst_table(self): - """Initialize the output mst table. - """ - create_query = """ - CREATE TABLE {self.model_selection_table} ( - {mst_key} SERIAL, - {model_id} INTEGER, - {compile_params} VARCHAR, - {fit_params} VARCHAR, - unique ({model_id}, {compile_params}, {fit_params}) - ); - """.format(self=self, - mst_key=ModelSelectionSchema.MST_KEY, - model_id=ModelSelectionSchema.MODEL_ID, - compile_params=ModelSelectionSchema.COMPILE_PARAMS, - fit_params=ModelSelectionSchema.FIT_PARAMS) - with MinWarning('warning'): - plpy.execute(create_query) - - def create_mst_summary_table(self): - """Initialize the output mst table. - """ - create_query = """ - CREATE TABLE {self.model_selection_summary_table} ( - {model_arch_table} VARCHAR - ); - """.format(self=self, - model_arch_table=ModelSelectionSchema.MODEL_ARCH_TABLE) - with MinWarning('warning'): - plpy.execute(create_query) - - def insert_into_mst_table(self): - """Insert every thing in self.msts into the mst table. - """ - for mst in self.msts: - model_id = mst[ModelSelectionSchema.MODEL_ID] - compile_params = mst[ModelSelectionSchema.COMPILE_PARAMS] - fit_params = mst[ModelSelectionSchema.FIT_PARAMS] - insert_query = """ - INSERT INTO - {self.model_selection_table}( - {model_id_col}, - {compile_params_col}, - {fit_params_col} - ) - VALUES ( - {model_id}, - $${compile_params}$$, - $${fit_params}$$ - ) - """.format(model_id_col=ModelSelectionSchema.MODEL_ID, - compile_params_col=ModelSelectionSchema.COMPILE_PARAMS, - fit_params_col=ModelSelectionSchema.FIT_PARAMS, - **locals()) - plpy.execute(insert_query) - insert_summary_query = """ - INSERT INTO - {self.model_selection_summary_table}( - {model_arch_table_name} - ) - VALUES ( - $${self.model_arch_table}$$ - ) - """.format(model_arch_table_name=ModelSelectionSchema.MODEL_ARCH_TABLE, - **locals()) - plpy.execute(insert_summary_query) diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in deleted file mode 100644 index c15757c..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.sql_in +++ /dev/null @@ -1,437 +0,0 @@ -/* ----------------------------------------------------------------------- *//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * - * @file madlib_keras_model_selection.sql_in - * - * @brief SQL functions for model hopper distributed training - * @date August 2019 - * - * - *//* ----------------------------------------------------------------------- */ - -m4_include(`SQLCommon.m4') -/** -@addtogroup grp_keras_setup_model_selection - -@brief Utility function to set up a model selection table for model architecture search -and hyperparameter tuning. - -\warning This MADlib method is still in early stage development. -Interface and implementation are subject to change. - - - -This utility function sets up a model selection table -for use by the multiple model Keras fit feature of MADlib. -By model selection we mean both hyperparameter tuning and -model architecture search. The table defines the unique combinations -of model architectures, compile and fit parameters -to run on a massively parallel processing database cluster. - -@anchor load_mst_table -@par Load Model Selection Table - -
-load_model_selection_table(
-    model_arch_table,
-    model_selection_table,
-    model_id_list,
-    compile_params_list,
-    fit_params_list
-    )
-
- -\b Arguments -
-
model_arch_table
-
VARCHAR. Table containing model architectures and weights. - For more information on this table - refer to Load Model. -
- -
model_selection_table
-
VARCHAR. Model selection table created by this utility. A summary table - named _summary is also created. Contents of both output - tables are described below. -
- -
model_id_list
-
INTEGER[]. Array of model IDs from the 'model_arch_table' to be included - in the run combinations. For hyperparameter search, this will typically be - one model ID. For model architecture search, this will be the different model IDs - that you want to test. -
- -
compile_params_list
-
VARCHAR[]. Array of compile parameters to be tested. Each element - of the array should consist of a string of compile parameters - exactly as it is to be passed to Keras. -
- -
fit_params_list
-
VARCHAR[]. Array of fit parameters to be tested. Each element - of the array should consist of a string of fit parameters - exactly as it is to be passed to Keras. -
- -
- -Output table -
- The model selection output table contains the following columns: - - - - - - - - - - - - - - - - - -
mst_keyINTEGER. ID that defines a unique tuple for - model architecture-compile parameters-fit parameters. -
model_idVARCHAR. Model architecture ID from the 'model_arch_table'. -
compile_paramsVARCHAR. Keras compile parameters. -
fit_paramsVARCHAR. Keras fit parameters. -
- A summary table named _summary is - also created, which contains the following column: - - - - - -
model_arch_tableVARCHAR. Name of the model architecture table containing the - model architecture IDs. -
-
- -@anchor example -@par Examples --# The model selection table works in conjunction with a model architecture table, -so we first create a model architecture table with two different models. Use Keras to define -a model architecture with 1 hidden layer: -
-import keras
-from keras.models import Sequential
-from keras.layers import Dense
-model1 = Sequential()
-model1.add(Dense(10, activation='relu', input_shape=(4,)))
-model1.add(Dense(10, activation='relu'))
-model1.add(Dense(3, activation='softmax'))
-model1.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type)                 Output Shape              Param #
-=================================================================
-dense_1 (Dense)              (None, 10)                50
-_________________________________________________________________
-dense_2 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_3 (Dense)              (None, 3)                 33
-=================================================================
-Total params: 193
-Trainable params: 193
-Non-trainable params: 0
-\endverbatim
-
-Export the model to JSON: -
-model1.to_json()
-
-
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-
-Now use Keras to define -a model architecture with 2 hidden layers: -
-model2 = Sequential()
-model2.add(Dense(10, activation='relu', input_shape=(4,)))
-model2.add(Dense(10, activation='relu'))
-model2.add(Dense(10, activation='relu'))
-model2.add(Dense(3, activation='softmax'))
-model2.summary()
-\verbatim
-
-_________________________________________________________________
-Layer (type)                 Output Shape              Param #
-=================================================================
-dense_4 (Dense)              (None, 10)                50
-_________________________________________________________________
-dense_5 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_6 (Dense)              (None, 10)                110
-_________________________________________________________________
-dense_7 (Dense)              (None, 3)                 33
-=================================================================
-Total params: 303
-Trainable params: 303
-Non-trainable params: 0
-\endverbatim
-
-Export the model to JSON: -
-model2.to_json()
-
-
-'{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}'
-
- --# Load both models into the architecture table: -
-DROP TABLE IF EXISTS model_arch_library;
-SELECT madlib.load_keras_model('model_arch_library',  -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json,         -- JSON blob
-                               NULL,                  -- Weights
-                               'Sophie',              -- Name
-                               'MLP with 1 hidden layer'       -- Descr
-);
-SELECT madlib.load_keras_model('model_arch_library',  -- Output table,
-$$
-{"class_name": "Sequential", "keras_version": "2.1.6", "config": [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], "backend": "tensorflow"}
-$$
-::json,         -- JSON blob
-                               NULL,                  -- Weights
-                               'Maria',               -- Name
-                               'MLP with 2 hidden layers'       -- Descr
-);
-SELECT model_id, name, description FROM model_arch_library ORDER BY model_id;
-
-
- model_id |  name  |       description
-----------+--------+--------------------------
-        1 | Sophie | MLP with 1 hidden layer
-        2 | Maria  | MLP with 2 hidden layers
-(2 rows)
-
- --# Load model selection table. Select the model(s) from the model -architecture table that you want to run, along with the compile and -fit parameters. Unique combinations will be created: -
-DROP TABLE IF EXISTS mst_table, mst_table_summary;
-SELECT madlib.load_model_selection_table('model_arch_library', -- model architecture table
-                                         'mst_table',          -- model selection table output
-                                          ARRAY[1,2],              -- model ids from model architecture table
-                                          ARRAY[                   -- compile params
-                                              $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$,
-                                              $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy']$$,
-                                              $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$
-                                          ],
-                                          ARRAY[                    -- fit params
-                                              $$batch_size=4,epochs=1$$,
-                                              $$batch_size=8,epochs=1$$
-                                          ]
-                                         );
-SELECT * FROM mst_table ORDER BY mst_key;
-
-
- mst_key | model_id      |                                 compile_params                                  |      fit_params
----------+---------------+---------------------------------------------------------------------------------+-----------------------
-       1 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1
-       2 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1
-       3 |             1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
-       4 |             1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
-       5 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
-       6 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-       7 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1
-       8 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1
-       9 |             2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1
-      10 |             2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1
-      11 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
-      12 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-(12 rows)
-
-The name of the model architecture table is stored in the summary table: -
-SELECT * FROM mst_table_summary;
-
-
-  model_arch_table
---------------------+
- model_arch_library
-(1 row)
-
- --# Create model selection table manually. If you would like to -have more control over the set of model selection parameters -to run, you can manually create the model selection table and -the associated summary table. Both must be created since -they are needed by the multiple model fit module. -For example, let's say we don't want all combinations -but only want batch_size=4 for model_id=1 and batch_size=8 for model_id=2: -
-DROP TABLE IF EXISTS mst_table_manual;
-CREATE TABLE mst_table_manual(
-    mst_key serial,
-    model_id integer,
-    compile_params varchar,
-    fit_params varchar
-);
-INSERT INTO mst_table_manual(model_id, compile_params, fit_params) VALUES
-(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
-(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
-(1, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$, 'batch_size=4,epochs=1'),
-(2, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$, 'batch_size=8,epochs=1'),
-(2, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy']$$, 'batch_size=8,epochs=1'),
-(2, $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$, 'batch_size=8,epochs=1');
-SELECT * FROM mst_table_manual ORDER BY mst_key;
-
-
- mst_key | model_id      |                                 compile_params                                  |      fit_params
----------+---------------+---------------------------------------------------------------------------------+-----------------------
-       1 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=4,epochs=1
-       2 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy']  | batch_size=4,epochs=1
-       3 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1
-       4 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']   | batch_size=8,epochs=1
-       5 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.01)',metrics=['accuracy']  | batch_size=8,epochs=1
-       6 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1
-(6 rows)
-
-Create the summary table which must be named -with the model selection output table appended by "_summary": -
-DROP TABLE IF EXISTS mst_table_manual_summary;
-CREATE TABLE mst_table_manual_summary (
-    model_arch_table varchar
-);
-INSERT INTO mst_table_manual_summary(model_arch_table) VALUES
-('model_arch_library');
-SELECT * FROM mst_table_manual_summary;
-
-
-  model_arch_table
---------------------+
- model_arch_library
-(1 row)
-
- --# Generate hyperparameters automatically. You can use other -libraries or methods to generate hyperparameters according -to the tests that you want to run. For example, let's randomly -generate batch size from powers of 2 and learning -rate on a log scale. -We use psycopg which is a PostgreSQL database adapter -for the Python programming language. -
-import numpy as np
-import psycopg2 as p2
-conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')
-#conn = p2.connect('postgresql://fmcquillan@localhost:5432/madlib')
-cur = conn.cursor()
-\#
-%sql DROP TABLE IF EXISTS mst_table_auto, mst_table_auto_summary;
-\#
-\#compile params
-learning_rate = np.random.permutation([0.1,0.01,0.001,0.0001])[:3]
-compile_param1 = "loss='categorical_crossentropy',optimizer='Adam(lr=" + str(learning_rate[0]) + ")',metrics=['accuracy']"
-compile_param2 = "loss='categorical_crossentropy',optimizer='Adam(lr=" + str(learning_rate[1]) + ")',metrics=['accuracy']"
-compile_param3 = "loss='categorical_crossentropy',optimizer='Adam(lr=" + str(learning_rate[2]) + ")',metrics=['accuracy']"
-compile_params = [compile_param1,compile_param2,compile_param3]
-\#
-\#fit params
-batch_size = np.random.permutation([4,8,16,32,64])[:2]
-fit_param1 = "batch_size=" + str(batch_size[0]) + ",epochs=1"
-fit_param2 = "batch_size=" + str(batch_size[1]) + ",epochs=1"
-fit_params = [fit_param1,fit_param2]
-\#
-query = "SELECT madlib.load_model_selection_table('model_arch_library', 'mst_table_auto', ARRAY[1,2], %s, %s);"
-\#
-cur.execute(query,[compile_params, fit_params])
-conn.commit()
-\#
-\# review model selection table
-%sql SELECT * FROM mst_table_auto ORDER BY mst_key;
-
-
- mst_key | model_id      |                                  compile_params                                  |      fit_params
----------+---------------+----------------------------------------------------------------------------------+-----------------------
-       1 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=4,epochs=1
-       2 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=8,epochs=1
-       3 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']    | batch_size=4,epochs=1
-       4 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']    | batch_size=8,epochs=1
-       5 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']  | batch_size=4,epochs=1
-       6 |             1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']  | batch_size=8,epochs=1
-       7 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=4,epochs=1
-       8 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.0001)',metrics=['accuracy'] | batch_size=8,epochs=1
-       9 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']    | batch_size=4,epochs=1
-      10 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']    | batch_size=8,epochs=1
-      11 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']  | batch_size=4,epochs=1
-      12 |             2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']  | batch_size=8,epochs=1
-(12 rows)
-
-The name of the model architecture table is stored in the summary table: -
-SELECT * FROM mst_table_auto_summary;
-
-
-  model_arch_table
---------------------+
- model_arch_library
-(1 row)
-
- -@anchor notes -@par Notes - -1. In this method, the same compile and fit parameters are applied to all model architectures -when generating combinations. However, you may wish to have different compile and fit parameters -for each model. To do so, call 'load_model_selection_table' -multiple times - once for each model. Then you can combine the resulting tables using UNION or other means. -Note that the 'mst_key' must be unique so you will need to regenerate it in your final combined table. - -@anchor related -@par Related Topics - -See keras_model_arch_table.sql_in - -*/ - -CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.load_model_selection_table( - model_arch_table VARCHAR, - model_selection_table VARCHAR, - model_id_list INTEGER[], - compile_params_list VARCHAR[], - fit_params_list VARCHAR[] -) RETURNS VOID AS $$ - PythonFunctionBodyOnly(`deep_learning', `madlib_keras_model_selection') - with AOControl(False): - mst_loader = madlib_keras_model_selection.MstLoader(**globals()) - mst_loader.load() -$$ LANGUAGE plpythonu VOLATILE -m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); - diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in deleted file mode 100644 index 412e63b..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_predict.py_in +++ /dev/null @@ -1,456 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import plpy - -import keras -from keras import backend as K -from keras.layers import * -from keras.models import * -from keras.optimizers import * - -from model_arch_info import * -from madlib_keras_helper import * -from madlib_keras_validator import * -from predict_input_params import PredictParamsProcessor -from utilities.control import MinWarning -from utilities.utilities import _assert -from utilities.utilities import add_postfix -from utilities.utilities import create_cols_from_array_sql_string -from utilities.utilities import get_segments_per_host -from utilities.utilities import unique_string -from utilities.validate_args import get_expr_type -from utilities.validate_args import input_tbl_valid - -from madlib_keras_wrapper import * - -class BasePredict(): - def __init__(self, schema_madlib, table_to_validate, test_table, id_col, - independent_varname, output_table, pred_type, use_gpus, module_name): - self.schema_madlib = schema_madlib - self.table_to_validate = table_to_validate - self.test_table = test_table - self.id_col = id_col - self.independent_varname = independent_varname - self.output_table = output_table - self.pred_type = pred_type - self.module_name = module_name - - self.segments_per_host = get_segments_per_host() - self.use_gpus = use_gpus if use_gpus else False - if self.use_gpus: - accessible_gpus_for_seg = get_accessible_gpus_for_seg(schema_madlib, self.segments_per_host, self.module_name) - _assert(len(set(accessible_gpus_for_seg)) == 1, - '{0}: Asymmetric gpu configurations are not supported'.format(self.module_name)) - self.gpus_per_host = accessible_gpus_for_seg[0] - else: - self.gpus_per_host = 0 - - self._set_default_pred_type() - - def _set_default_pred_type(self): - self.pred_type = 'response' if not self.pred_type else self.pred_type - self.is_response = True if self.pred_type == 'response' else False - - def call_internal_keras(self): - if self.is_response: - pred_col_name = add_postfix("estimated_", self.dependent_varname) - pred_col_type = self.dependent_vartype - else: - pred_col_name = "prob" - pred_col_type = 'double precision' - - intermediate_col = unique_string() - class_values = strip_trailing_nulls_from_class_values(self.class_values) - - prediction_select_clause, create_table_columns = create_cols_from_array_sql_string( - class_values, intermediate_col, pred_col_name, - pred_col_type, self.is_response, self.module_name) - gp_segment_id_col, seg_ids_test, \ - images_per_seg_test = get_image_count_per_seg_for_non_minibatched_data_from_db( - self.test_table) - segments_per_host = get_segments_per_host() - - select_segmentid_comma = "" - group_by_clause = "" - join_cond_on_segmentid = "" - if not is_platform_pg(): - select_segmentid_comma = "{self.test_table}.gp_segment_id AS gp_segment_id,".format(self=self) - group_by_clause = "GROUP BY {self.test_table}.gp_segment_id".format(self=self) - join_cond_on_segmentid = "{self.test_table}.gp_segment_id=min_ctid.gp_segment_id AND".format(self=self) - - # Calling CREATE TABLE instead of CTAS, to ensure that the plan_cache_mode - # guc codepath is called when passing in the weights - plpy.execute(""" - CREATE TABLE {self.output_table} - ({self.id_col} {self.id_col_type}, {create_table_columns}) - """.format(self=self, create_table_columns=create_table_columns)) - # Passing huge model weights to internal_keras_predict() for each row - # resulted in slowness of overall madlib_keras_predict(). - # To avoid this, a CASE is added to pass the model weights only for - # the very first row(min(ctid)) that is fetched on each segment and NULL - # for the other rows. - predict_query = plpy.prepare(""" - INSERT INTO {self.output_table} - SELECT {self.id_col}::{self.id_col_type}, {prediction_select_clause} - FROM ( - SELECT {self.test_table}.{self.id_col}, - ({self.schema_madlib}.internal_keras_predict - ({self.independent_varname}, - $1, - CASE WHEN {self.test_table}.ctid = min_ctid.ctid THEN $2 ELSE NULL END, - {self.is_response}, - {self.normalizing_const}, - {gp_segment_id_col}, - ARRAY{seg_ids_test}, - ARRAY{images_per_seg_test}, - {self.use_gpus}, - {self.gpus_per_host}, - {segments_per_host}) - ) AS {intermediate_col} - FROM {self.test_table} - LEFT JOIN - (SELECT {select_segmentid_comma} MIN({self.test_table}.ctid) AS ctid - FROM {self.test_table} - {group_by_clause}) min_ctid - ON {join_cond_on_segmentid} {self.test_table}.ctid=min_ctid.ctid - ) q - """.format(self=self, prediction_select_clause=prediction_select_clause, - seg_ids_test=seg_ids_test, - images_per_seg_test=images_per_seg_test, - gp_segment_id_col=gp_segment_id_col, - segments_per_host=segments_per_host, - intermediate_col=intermediate_col, - select_segmentid_comma=select_segmentid_comma, - group_by_clause=group_by_clause, - join_cond_on_segmentid=join_cond_on_segmentid), - ["text", "bytea"]) - plpy.execute(predict_query, [self.model_arch, self.model_weights]) - - def set_default_class_values(self, class_values): - self.class_values = class_values - if self.pred_type == 'prob': - return - if self.class_values is None: - num_classes = get_num_classes(self.model_arch) - self.class_values = range(0, num_classes) - -@MinWarning("warning") -class Predict(BasePredict): - def __init__(self, schema_madlib, model_table, - test_table, id_col, independent_varname, - output_table, pred_type, use_gpus, - mst_key, **kwargs): - - self.module_name = 'madlib_keras_predict' - self.model_table = model_table - self.mst_key = mst_key - self.is_mult_model = mst_key is not None - if self.model_table: - self.model_summary_table = add_postfix(self.model_table, "_summary") - - BasePredict.__init__(self, schema_madlib, model_table, test_table, - id_col, independent_varname, - output_table, pred_type, - use_gpus, self.module_name) - param_proc = PredictParamsProcessor(self.model_table, self.module_name, self.mst_key) - if self.is_mult_model: - self.temp_summary_view = param_proc.model_summary_table - self.model_summary_table = self.temp_summary_view - self.dependent_vartype = param_proc.get_dependent_vartype() - self.model_weights = param_proc.get_model_weights() - self.model_arch = param_proc.get_model_arch() - class_values = param_proc.get_class_values() - self.set_default_class_values(class_values) - self.normalizing_const = param_proc.get_normalizing_const() - self.dependent_varname = param_proc.get_dependent_varname() - - self.validate() - self.id_col_type = get_expr_type(self.id_col, self.test_table) - BasePredict.call_internal_keras(self) - if self.is_mult_model: - plpy.execute("DROP VIEW IF EXISTS {}".format(self.temp_summary_view)) - - def validate(self): - input_tbl_valid(self.model_table, self.module_name) - if self.is_mult_model and not columns_exist_in_table(self.model_table, ['mst_key']): - plpy.error("{self.module_name}: Single model should not pass mst_key".format(**locals())) - if not self.is_mult_model and columns_exist_in_table(self.model_table, ['mst_key']): - plpy.error("{self.module_name}: Multi-model needs to pass mst_key".format(**locals())) - InputValidator.validate_predict_evaluate_tables( - self.module_name, self.model_table, self.model_summary_table, - self.test_table, self.output_table, self.independent_varname) - - InputValidator.validate_id_in_test_tbl( - self.module_name, self.test_table, self.id_col) - - InputValidator.validate_class_values( - self.module_name, self.class_values, self.pred_type, self.model_arch) - input_shape = get_input_shape(self.model_arch) - InputValidator.validate_pred_type( - self.module_name, self.pred_type, self.class_values) - InputValidator.validate_input_shape( - self.test_table, self.independent_varname, input_shape, 1) - -@MinWarning("warning") -class PredictBYOM(BasePredict): - def __init__(self, schema_madlib, model_arch_table, model_id, - test_table, id_col, independent_varname, output_table, - pred_type, use_gpus, class_values, normalizing_const, - **kwargs): - - self.module_name='madlib_keras_predict_byom' - self.model_arch_table = model_arch_table - self.model_id = model_id - self.class_values = class_values - self.normalizing_const = normalizing_const - self.dependent_varname = 'dependent_var' - BasePredict.__init__(self, schema_madlib, model_arch_table, - test_table, id_col, independent_varname, - output_table, pred_type, use_gpus, self.module_name) - if self.is_response: - self.dependent_vartype = 'text' - else: - self.dependent_vartype = 'double precision' - ## Set default values for norm const and class_values - # use_gpus and pred_type are defaulted in base_predict's init - self.normalizing_const = normalizing_const - if self.normalizing_const is None: - self.normalizing_const = DEFAULT_NORMALIZING_CONST - InputValidator.validate_predict_byom_tables( - self.module_name, self.model_arch_table, self.model_id, - self.test_table, self.id_col, self.output_table, - self.independent_varname) - self.validate_and_set_defaults() - self.id_col_type = get_expr_type(self.id_col, self.test_table) - BasePredict.call_internal_keras(self) - - def validate_and_set_defaults(self): - # Set some defaults first and then validate and then set some more defaults - self.model_arch, self.model_weights = get_model_arch_weights( - self.model_arch_table, self.model_id) - # Assert model_weights and model_arch are not empty. - _assert(self.model_weights and self.model_arch, - "{0}: Model weights and architecture should not be NULL.".format( - self.module_name)) - self.set_default_class_values(self.class_values) - - InputValidator.validate_pred_type( - self.module_name, self.pred_type, self.class_values) - InputValidator.validate_normalizing_const( - self.module_name, self.normalizing_const) - InputValidator.validate_class_values( - self.module_name, self.class_values, self.pred_type, self.model_arch) - InputValidator.validate_input_shape( - self.test_table, self.independent_varname, - get_input_shape(self.model_arch), 1) - -def internal_keras_predict(independent_var, model_architecture, model_weights, - is_response, normalizing_const, current_seg_id, seg_ids, - images_per_seg, use_gpus, gpus_per_host, segments_per_host, - **kwargs): - SD = kwargs['SD'] - model_key = 'segment_model_predict' - row_count_key = 'row_count' - try: - device_name = get_device_name_and_set_cuda_env( gpus_per_host, current_seg_id) - if model_key not in SD: - set_keras_session(device_name, gpus_per_host, segments_per_host) - model = model_from_json(model_architecture) - set_model_weights(model, model_weights) - SD[model_key] = model - SD[row_count_key] = 0 - else: - model = SD[model_key] - SD[row_count_key] += 1 - - # Since the test data isn't mini-batched, - # we have to make sure that the test data np array has the same - # number of dimensions as input_shape. So we add a dimension to x. - independent_var = expand_input_dims(independent_var) - independent_var /= normalizing_const - - if is_response: - with K.tf.device(device_name): - y_prob = model.predict(independent_var) - proba_argmax = y_prob.argmax(axis=-1) - # proba_argmax is a list with exactly one element in it. That element - # refers to the index containing the largest probability value in the - # output of Keras' predict function. - result = proba_argmax - else: - with K.tf.device(device_name): - probs = model.predict(independent_var) - # probs is a list containing a list of probability values, of all - # class levels. Since we are assuming each input is a single image, - # and not mini-batched, this list contains exactly one list in it, - # so return back the first list in probs. - result = probs[0] - total_images = get_image_count_per_seg_from_array(seg_ids.index(current_seg_id), - images_per_seg) - - if SD[row_count_key] == total_images: - SD.pop(model_key, None) - SD.pop(row_count_key, None) - clear_keras_session() - return result - except Exception as ex: - SD.pop(model_key, None) - SD.pop(row_count_key, None) - clear_keras_session() - plpy.error(ex) - - -def predict_help(schema_madlib, message, **kwargs): - """ - Help function for keras predict - - Args: - @param schema_madlib - @param message: string, Help message string - @param kwargs - - Returns: - String. Help/usage information - """ - if not message: - help_string = """ ------------------------------------------------------------------------ - SUMMARY ------------------------------------------------------------------------ -This function allows the user to predict using a madlib_keras_fit trained -model. - -For more details on function usage: - SELECT {schema_madlib}.madlib_keras_predict('usage') - """ - elif message in ['usage', 'help', '?']: - help_string = """ ------------------------------------------------------------------------ - USAGE ------------------------------------------------------------------------ - SELECT {schema_madlib}.madlib_keras_predict( - model_table, -- Name of the table containing the model - test_table, -- Name of the table containing the evaluation dataset - id_col, -- Name of the id column in the test data table - independent_varname, -- Name of the column with independent - variables in the test table - output_table, -- Name of the output table - pred_type, -- The type of the desired output - use_gpus, -- Flag for enabling GPU support - mst_key -- Identifier for the desired model out of multimodel - training output - ) - ); - ------------------------------------------------------------------------ - OUTPUT ------------------------------------------------------------------------ -The output table ('output_table' above) contains the following columns: - -id: Gives the 'id' for each prediction, corresponding - to each row from the test_table. -estimated_COL_NAME: (For pred_type='response') The estimated class for - classification, where COL_NAME is the name of the - column to be predicted from test data. -prob_CLASS: (For pred_type='prob' for classification) The - probability of a given class. There will be one - column for each class in the training data. - TODO change this -""" - else: - help_string = "No such option. Use {schema_madlib}.madlib_keras_predict()" - - return help_string.format(schema_madlib=schema_madlib) - -def predict_byom_help(schema_madlib, message, **kwargs): - """ - Help function for keras predict - - Args: - @param schema_madlib - @param message: string, Help message string - @param kwargs - - Returns: - String. Help/usage information - """ - if not message: - help_string = """ ------------------------------------------------------------------------ - SUMMARY ------------------------------------------------------------------------ -This function allows the user to predict with their own pre trained model (note -that this model doesn't have to be trained using MADlib.) - -For more details on function usage: - SELECT {schema_madlib}.madlib_keras_predict_byom('usage') - """ - elif message in ['usage', 'help', '?']: - help_string = """ ------------------------------------------------------------------------ - USAGE ------------------------------------------------------------------------ - SELECT {schema_madlib}.madlib_keras_predict_byom( - model_arch_table, -- Name of the table containing the model architecture - and the pre trained model weights - model_id, -- This is the id in 'model_arch_table' containing the - model architecture - test_table, -- Name of the table containing the evaluation dataset - id_col, -- Name of the id column in the test data table - independent_varname, -- Name of the column with independent - variables in the test table - output_table, -- Name of the output table - pred_type, -- The type of the desired output - use_gpus, -- Flag for enabling GPU support - class_values, -- List of class labels that were used while training the - model. If class_values is passed in as NULL, the output - table will have a column named 'prob' which is an array - of probabilities of all the classes. - Otherwise if class_values is not NULL, then the output - table will contain a column for each class/label from - the training data - normalizing_const, -- Normalizing constant used for standardizing arrays in - independent_varname - ) - ); - ------------------------------------------------------------------------ - OUTPUT ------------------------------------------------------------------------ -The output table ('output_table' above) contains the following columns: - -id: Gives the 'id' for each prediction, corresponding - to each row from the test_table. -estimated_dependent_var: (For pred_type='response') The estimated class for - classification. If class_values is passed in as NULL, then we - assume that the class labels are [0,1,2...,n] where n in the - num of classes in the model architecture. -prob_CLASS: (For pred_type='prob' for classification) The - probability of a given class. - If class_values is passed in as NULL, we create just one column - called 'prob' which is an array of probabilites of all the classes - Otherwise if class_values is not NULL, then there will be one - column for each class in the training data. -""" - else: - help_string = "No such option. Use {schema_madlib}.madlib_keras_predict_byom()" - - return help_string.format(schema_madlib=schema_madlib) -# --------------------------------------------------------------------- diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_serializer.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_serializer.py_in deleted file mode 100644 index 6fa210c..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_serializer.py_in +++ /dev/null @@ -1,162 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import numpy as np -from utilities.utilities import _assert - -# TODO -# 1. Current serializing logic - # serialized string -> byte string - # np.array(np.array(image_count).concatenate(weights_np_array)).tostring() - # Proposed logic - # image_count can be a separate value - # weights -> np.array(weights).tostring() - # combine these 2 into one string by a random splitter - # serialized string -> imagecount_splitter_weights -# 2. combine the serialize_state_with_nd_weights and serialize_state_with_1d_weights - # into one function called serialize_state. This function can infer the shape - # of the model weights and then flatten if they are nd weights. -# 3. Same as 2 for deserialize - - -""" -workflow -1. Set initial weights in madlib keras fit function. -2. Serialize these initial model weights as a byte string and pass it to keras step -3. Deserialize the state passed from the previous step into a list of nd weights -that will be passed on to model.set_weights() -4. At the end of each buffer in fit transition, serialize the image count and -the model weights into a bytestring that will be passed on to the fit merge function. -5. In fit merge, deserialize the state as image and 1d np arrays. Do some averaging -operations and serialize them again into a state which contains the image -and the 1d state. same for fit final -6. Return the final state from fit final to fit which will then be deserialized -as 1d weights to be passed on to the evaluate function -""" -def get_serialized_1d_weights_from_state(state): - """ - Output of this function is used to deserialize the output of each iteration - of madlib keras step UDA. - - :param state: bytestring serialized model state containing image count - and weights - :return: model weights serialized as bytestring - """ - _ , weights = deserialize_as_image_1d_weights(state) - return weights.tostring() - -def serialize_state_with_nd_weights(image_count, model_weights): - """ - This function is called when the output of keras.get_weights() (list of nd - np arrays) has to be converted into a serialized model state. - - :param image_count: float value - :param model_weights: a list of numpy arrays, what you get from - keras.get_weights() - :return: Image count and model weights serialized into a bytestring format - - """ - if model_weights is None: - return None - flattened_weights = [w.flatten() for w in model_weights] - state = [np.array([image_count])] + flattened_weights - state = np.concatenate(state) - return np.float32(state).tostring() - - -def serialize_state_with_1d_weights(image_count, model_weights): - """ - This function is called when the weights are to be passed to the keras fit - merge and final functions. - - :param image_count: float value - :param model_weights: a single flattened numpy array containing all of the - weights - :return: Image count and model weights serialized into a bytestring format - - """ - if model_weights is None: - return None - merge_state = np.array([image_count]) - merge_state = np.concatenate((merge_state, model_weights)) - merge_state = np.float32(merge_state) - return merge_state.tostring() - - -def deserialize_as_image_1d_weights(state): - """ - This function is called when the model state needs to be deserialized in - the keras fit merge and final functions. - - :param state: the stringified (serialized) state containing image_count and - model_weights - :return: - image_count: total buffer counts processed - model_weights: a single flattened numpy array containing all of the - weights - """ - if not state: - return None - state = np.fromstring(state, dtype=np.float32) - return float(state[0]), state[1:] - - -def serialize_nd_weights(model_weights): - """ - This function is called for passing the initial model weights from the keras - fit function to the keras fit transition function. - :param model_weights: a list of numpy arrays, what you get from - keras.get_weights() - :return: Model weights serialized into a bytestring format - """ - if model_weights is None: - return None - flattened_weights = [w.flatten() for w in model_weights] - flattened_weights = np.concatenate(flattened_weights) - return np.float32(flattened_weights).tostring() - - -def deserialize_as_nd_weights(model_weights_serialized, model_shapes): - """ - The output of this function is used to set keras model weights using the - function model.set_weights() - :param model_weights_serialized: bytestring containing model weights - :param model_shapes: list containing the shapes of each layer. - :return: list of nd numpy arrays containing all of the - weights - """ - if not model_weights_serialized or not model_shapes: - return None - - i, j, model_weights = 0, 0, [] - model_weights_serialized = np.fromstring(model_weights_serialized, dtype=np.float32) - - total_model_shape = \ - sum([reduce(lambda x, y: x * y, ls) for ls in model_shapes]) - total_weights_shape = model_weights_serialized.size - _assert(total_model_shape == total_weights_shape, - "Number of elements in model weights({0}) doesn't match model({1})."\ - .format(total_weights_shape, total_model_shape)) - while j < len(model_shapes): - next_pointer = i + reduce(lambda x, y: x * y, model_shapes[j]) - weight_arr_portion = model_weights_serialized[i:next_pointer] - model_weights.append(np.array(weight_arr_portion).reshape(model_shapes[j])) - i, j = next_pointer, j + 1 - return model_weights - - diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in deleted file mode 100644 index 11730cf..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in +++ /dev/null @@ -1,506 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import plpy -from keras_model_arch_table import ModelArchSchema -from model_arch_info import get_num_classes -from madlib_keras_helper import CLASS_VALUES_COLNAME -from madlib_keras_helper import COMPILE_PARAMS_COLNAME -from madlib_keras_helper import DEPENDENT_VARNAME_COLNAME -from madlib_keras_helper import DEPENDENT_VARTYPE_COLNAME -from madlib_keras_helper import MODEL_ID_COLNAME -from madlib_keras_helper import MODEL_ARCH_TABLE_COLNAME -from madlib_keras_helper import MODEL_WEIGHTS_COLNAME -from madlib_keras_helper import NORMALIZING_CONST_COLNAME -from madlib_keras_helper import DISTRIBUTION_KEY_COLNAME -from madlib_keras_helper import METRIC_TYPE_COLNAME -from madlib_keras_helper import INTERNAL_GPU_CONFIG -from madlib_keras_helper import query_model_configs - -from utilities.minibatch_validation import validate_bytea_var_for_minibatch -from utilities.utilities import _assert -from utilities.utilities import add_postfix -from utilities.utilities import is_platform_pg -from utilities.utilities import is_var_valid -from utilities.validate_args import cols_in_tbl_valid -from utilities.validate_args import columns_exist_in_table -from utilities.validate_args import get_expr_type -from utilities.validate_args import input_tbl_valid -from utilities.validate_args import output_tbl_valid -from madlib_keras_wrapper import parse_and_validate_fit_params -from madlib_keras_wrapper import parse_and_validate_compile_params - -class InputValidator: - @staticmethod - def validate_predict_evaluate_tables( - module_name, model_table, model_summary_table, test_table, output_table, - independent_varname): - InputValidator._validate_model_weights_tbl(module_name, model_table) - InputValidator._validate_model_summary_tbl( - module_name, model_summary_table) - InputValidator._validate_test_tbl( - module_name, test_table, independent_varname) - output_tbl_valid(output_table, module_name) - - @staticmethod - def validate_id_in_test_tbl(module_name, test_table, id_col): - _assert(is_var_valid(test_table, id_col), - "{module_name} error: invalid id column " - "('{id_col}') for test table ({table}).".format( - module_name=module_name, - id_col=id_col, - table=test_table)) - - @staticmethod - def validate_predict_byom_tables(module_name, model_arch_table, model_id, - test_table, id_col, output_table, - independent_varname): - InputValidator.validate_model_arch_table( - module_name, model_arch_table, model_id) - InputValidator._validate_test_tbl( - module_name, test_table, independent_varname) - InputValidator.validate_id_in_test_tbl(module_name, test_table, id_col) - - output_tbl_valid(output_table, module_name) - - - @staticmethod - def validate_pred_type(module_name, pred_type, class_values): - if not pred_type in ['prob', 'response']: - plpy.error("{0}: Invalid value for pred_type param ({1}). Must be "\ - "either response or prob.".format(module_name, pred_type)) - - @staticmethod - def validate_input_shape(table, independent_varname, input_shape, offset, - is_minibatched=False): - """ - Validate if the input shape specified in model architecture is the same - as the shape of the image specified in the indepedent var of the input - table. - offset: This offset is the index of the start of the image array. We also - need to consider that sql array indexes start from 1 - For ex if the image is of shape [32,32,3] and is minibatched, the image will - look like [10, 32, 32, 3]. The offset in this case is 1 (start the index at 1) + - 1 (ignore the buffer size 10) = 2. - If the image is not batched then it will look like [32, 32 ,3] and the offset in - this case is 1 (start the index at 1). - """ - - if is_minibatched: - ind_shape_col = add_postfix(independent_varname, "_shape") - query = """ - SELECT {ind_shape_col} AS shape - FROM {table} - LIMIT 1 - """.format(**locals()) - # This query will fail if an image in independent var does not have the - # same number of dimensions as the input_shape. - result = plpy.execute(query)[0]['shape'] - result = result[1:] - else: - array_upper_query = ", ".join("array_upper({0}, {1}) AS n_{2}".format( - independent_varname, i+offset, i) for i in range(len(input_shape))) - query = """ - SELECT {0} - FROM {1} - LIMIT 1 - """.format(array_upper_query, table) - - # This query will fail if an image in independent var does not have the - # same number of dimensions as the input_shape. - result = plpy.execute(query)[0] - - _assert(len(result) == len(input_shape), - "model_keras error: The number of dimensions ({0}) of each image" - " in model architecture and {1} in {2} ({3}) do not match.".format( - len(input_shape), independent_varname, table, len(result))) - - for i in range(len(input_shape)): - if is_minibatched: - key_name = i - input_shape_from_table = [result[j] - for j in range(len(input_shape))] - else: - key_format = "n_{0}" - key_name = key_format.format(i) - input_shape_from_table = [result[key_format.format(j)] - for j in range(len(input_shape))] - - if result[key_name] != input_shape[i]: - # Construct the shape in independent varname to display - # meaningful error msg. - plpy.error("model_keras error: Input shape {0} in the model" - " architecture does not match the input shape {1} of column" - " {2} in table {3}.".format( - input_shape, input_shape_from_table, - independent_varname, table)) - - @staticmethod - def validate_model_arch_table(module_name, model_arch_table, model_id): - input_tbl_valid(model_arch_table, module_name) - _assert(model_id is not None, - "{0}: Invalid model architecture ID.".format(module_name)) - - - @staticmethod - def validate_normalizing_const(module_name, normalizing_const): - _assert(normalizing_const > 0, - "{0} error: Normalizing constant has to be greater than 0.". - format(module_name)) - - @staticmethod - def validate_class_values(module_name, class_values, pred_type, model_arch): - if not class_values: - return - num_classes = len(class_values) - _assert(num_classes == get_num_classes(model_arch), - "{0}: The number of class values do not match the " \ - "provided architecture.".format(module_name)) - if pred_type == 'prob' and num_classes+1 >= 1600: - plpy.error({"{0}: The output will have {1} columns, exceeding the "\ - " max number of columns that can be created (1600)".format( - module_name, num_classes+1)}) - - @staticmethod - def validate_model_weights(module_name, model_arch, model_weights): - _assert(model_weights and model_arch, - "{0}: Model weights and architecture must be valid.".format( - module_name)) - - @staticmethod - def _validate_model_weights_tbl(module_name, model_table): - _assert(is_var_valid(model_table, MODEL_WEIGHTS_COLNAME), - "{module_name} error: column '{model_weights}' " - "does not exist in model table '{table}'.".format( - module_name=module_name, - model_weights=MODEL_WEIGHTS_COLNAME, - table=model_table)) - _assert(is_var_valid(model_table, ModelArchSchema.MODEL_ARCH), - "{module_name} error: column '{model_arch}' " - "does not exist in model table '{table}'.".format( - module_name=module_name, - model_arch=ModelArchSchema.MODEL_ARCH, - table=model_table)) - - @staticmethod - def _validate_test_tbl(module_name, test_table, independent_varname): - input_tbl_valid(test_table, module_name) - _assert(is_var_valid(test_table, independent_varname), - "{module_name} error: invalid independent_varname " - "('{independent_varname}') for test table " - "({table}).".format( - module_name=module_name, - independent_varname=independent_varname, - table=test_table)) - - @staticmethod - def _validate_model_summary_tbl(module_name, model_summary_table): - input_tbl_valid(model_summary_table, module_name) - cols_to_check_for = [CLASS_VALUES_COLNAME, - DEPENDENT_VARNAME_COLNAME, - DEPENDENT_VARTYPE_COLNAME, - MODEL_ID_COLNAME, - MODEL_ARCH_TABLE_COLNAME, - NORMALIZING_CONST_COLNAME, - COMPILE_PARAMS_COLNAME, - METRIC_TYPE_COLNAME] - _assert(columns_exist_in_table( - model_summary_table, cols_to_check_for), - "{0} error: One or more expected columns missing in model " - "summary table ('{1}'). The expected columns are {2}.".format( - module_name, model_summary_table, cols_to_check_for)) - - @staticmethod - def _validate_gpu_config(module_name, source_table, accessible_gpus_for_seg): - - summary_table = add_postfix(source_table, "_summary") - gpu_config = plpy.execute( - "SELECT {0} FROM {1}".format(INTERNAL_GPU_CONFIG, summary_table) - )[0][INTERNAL_GPU_CONFIG] - if gpu_config == 'all_segments': - _assert(0 not in accessible_gpus_for_seg, - "{0} error: Host(s) are missing gpus.".format(module_name)) - else: - for i in gpu_config: - _assert(accessible_gpus_for_seg[i] != 0, - "{0} error: Segment {1} does not have gpu".format(module_name, i)) - -class FitCommonValidator(object): - def __init__(self, source_table, validation_table, output_model_table, - model_arch_table, model_id, dependent_varname, - independent_varname, num_iterations, - metrics_compute_frequency, warm_start, - use_gpus, accessible_gpus_for_seg, module_name): - self.source_table = source_table - self.validation_table = validation_table - self.output_model_table = output_model_table - self.model_arch_table = model_arch_table - self.model_id = model_id - self.dependent_varname = dependent_varname - self.independent_varname = independent_varname - self.dep_shape_col = add_postfix(dependent_varname, "_shape") - self.ind_shape_col = add_postfix(independent_varname, "_shape") - self.metrics_compute_frequency = metrics_compute_frequency - self.warm_start = warm_start - self.num_iterations = num_iterations - self.source_summary_table = None - if self.source_table: - self.source_summary_table = add_postfix( - self.source_table, "_summary") - if self.output_model_table: - self.output_summary_model_table = add_postfix( - self.output_model_table, "_summary") - self.accessible_gpus_for_seg = accessible_gpus_for_seg - self.module_name = module_name - self._validate_common_args() - if use_gpus: - InputValidator._validate_gpu_config(self.module_name, - self.source_table, self.accessible_gpus_for_seg) - - def _validate_common_args(self): - _assert(self.num_iterations > 0, - "{0}: Number of iterations cannot be < 1.".format(self.module_name)) - _assert(self._is_valid_metrics_compute_frequency(), - "{0}: metrics_compute_frequency must be in the range (1 - {1}).".format( - self.module_name, self.num_iterations)) - input_tbl_valid(self.source_table, self.module_name) - input_tbl_valid(self.source_summary_table, self.module_name, - error_suffix_str="Please ensure that the source table ({0}) " - "has been preprocessed by " - "the image preprocessor.".format(self.source_table)) - cols_in_tbl_valid(self.source_summary_table, [CLASS_VALUES_COLNAME, - NORMALIZING_CONST_COLNAME, DEPENDENT_VARTYPE_COLNAME, - 'dependent_varname', 'independent_varname'], self.module_name) - if not is_platform_pg(): - cols_in_tbl_valid(self.source_table, [DISTRIBUTION_KEY_COLNAME], self.module_name) - - # Source table and validation tables must have the same schema - self._validate_input_table(self.source_table) - validate_bytea_var_for_minibatch(self.source_table, - self.dependent_varname) - - self._validate_validation_table() - if self.warm_start: - input_tbl_valid(self.output_model_table, self.module_name) - input_tbl_valid(self.output_summary_model_table, self.module_name) - else: - output_tbl_valid(self.output_model_table, self.module_name) - output_tbl_valid(self.output_summary_model_table, self.module_name) - - def _validate_input_table(self, table): - _assert(is_var_valid(table, self.independent_varname), - "{module_name}: invalid independent_varname " - "('{independent_varname}') for table ({table}). " - "Please ensure that the input table ({table}) " - "has been preprocessed by the image preprocessor.".format( - module_name=self.module_name, - independent_varname=self.independent_varname, - table=table)) - - _assert(is_var_valid(table, self.dependent_varname), - "{module_name}: invalid dependent_varname " - "('{dependent_varname}') for table ({table}). " - "Please ensure that the input table ({table}) " - "has been preprocessed by the image preprocessor.".format( - module_name=self.module_name, - dependent_varname=self.dependent_varname, - table=table)) - - _assert(is_var_valid(table, self.ind_shape_col), - "{module_name}: invalid independent_var_shape " - "('{ind_shape_col}') for table ({table}). " - "Please ensure that the input table ({table}) " - "has been preprocessed by the image preprocessor.".format( - module_name=self.module_name, - ind_shape_col=self.ind_shape_col, - table=table)) - - _assert(is_var_valid(table, self.dep_shape_col), - "{module_name}: invalid dependent_var_shape " - "('{dep_shape_col}') for table ({table}). " - "Please ensure that the input table ({table}) " - "has been preprocessed by the image preprocessor.".format( - module_name=self.module_name, - dep_shape_col=self.dep_shape_col, - table=table)) - - if not is_platform_pg(): - _assert(is_var_valid(table, DISTRIBUTION_KEY_COLNAME), - "{module_name}: missing distribution key " - "('{dist_key_col}') for table ({table}). " - "Please ensure that the input table ({table}) " - "has been preprocessed by the image preprocessor.".format( - module_name=self.module_name, - dist_key_col=DISTRIBUTION_KEY_COLNAME, - table=table)) - - def _is_valid_metrics_compute_frequency(self): - return self.metrics_compute_frequency is None or \ - (self.metrics_compute_frequency >= 1 and \ - self.metrics_compute_frequency <= self.num_iterations) - - def _validate_validation_table(self): - if self.validation_table and self.validation_table.strip() != '': - input_tbl_valid(self.validation_table, self.module_name) - self._validate_input_table(self.validation_table) - dependent_vartype = get_expr_type(self.dependent_varname, - self.validation_table) - _assert(dependent_vartype == 'bytea', - "Dependent variable column {0} in validation table {1} should be " - "a bytea and also one hot encoded.".format( - self.dependent_varname, self.validation_table)) - - - def validate_input_shapes(self, input_shape): - InputValidator.validate_input_shape(self.source_table, self.independent_varname, - input_shape, 2, True) - if self.validation_table: - InputValidator.validate_input_shape( - self.validation_table, self.independent_varname, - input_shape, 2, True) - - -class FitInputValidator(FitCommonValidator): - def __init__(self, source_table, validation_table, output_model_table, - model_arch_table, model_id, dependent_varname, - independent_varname, num_iterations, - metrics_compute_frequency, warm_start, - use_gpus, accessible_gpus_for_seg): - - self.module_name = 'madlib_keras_fit' - super(FitInputValidator, self).__init__(source_table, - validation_table, - output_model_table, - model_arch_table, - model_id, - dependent_varname, - independent_varname, - num_iterations, - metrics_compute_frequency, - warm_start, - use_gpus, - accessible_gpus_for_seg, - self.module_name) - InputValidator.validate_model_arch_table(self.module_name, self.model_arch_table, - self.model_id) - -class FitMultipleInputValidator(FitCommonValidator): - def __init__(self, source_table, validation_table, output_model_table, - model_selection_table, model_selection_summary_table, dependent_varname, - independent_varname, num_iterations, model_info_table, mst_key_col, - model_arch_table_col, metrics_compute_frequency, warm_start, - use_gpus, accessible_gpus_for_seg): - - self.module_name = 'madlib_keras_fit_multiple' - - input_tbl_valid(model_selection_table, self.module_name) - input_tbl_valid(model_selection_summary_table, self.module_name, - error_suffix_str="Please ensure that the model selection table ({0}) " - "has been created by " - "load_model_selection_table().".format( - model_selection_table)) - self.msts, self.model_arch_table = query_model_configs( - model_selection_table, model_selection_summary_table, - mst_key_col, model_arch_table_col) - if warm_start: - input_tbl_valid(model_info_table, self.module_name) - else: - output_tbl_valid(model_info_table, self.module_name) - super(FitMultipleInputValidator, self).__init__(source_table, - validation_table, - output_model_table, - self.model_arch_table, - None, - dependent_varname, - independent_varname, - num_iterations, - metrics_compute_frequency, - warm_start, - use_gpus, - accessible_gpus_for_seg, - self.module_name) - -class MstLoaderInputValidator(): - def __init__(self, - model_arch_table, - model_selection_table, - model_selection_summary_table, - model_id_list, - compile_params_list, - fit_params_list - ): - self.model_arch_table = model_arch_table - self.model_selection_table = model_selection_table - self.model_selection_summary_table = model_selection_summary_table - self.model_id_list = model_id_list - self.compile_params_list = compile_params_list - self.fit_params_list = fit_params_list - self.module_name = 'load_model_selection_table' - self._validate_input_args() - - def _validate_input_args(self): - self._validate_input_output_tables() - self._validate_model_ids() - self._validate_compile_and_fit_params() - - def _validate_model_ids(self): - model_id_str = '({0})'\ - .format(','.join([str(x) for x in self.model_id_list])) - query = """ - SELECT count(model_id) - FROM {self.model_arch_table} - WHERE model_id IN {model_id_str} - """.format(**locals()) - res = int(plpy.execute(query)[0]['count']) - _assert( - res == len(self.model_id_list), - "{0}: One or more model_id of {1} not found in table {2}".format( - self.module_name, - model_id_str, - self.model_arch_table - ) - ) - def _validate_compile_and_fit_params(self): - if not self.fit_params_list: - plpy.error("fit_params_list cannot be NULL") - for fit_params in self.fit_params_list: - try: - res = parse_and_validate_fit_params(fit_params) - except Exception as e: - plpy.error( - """Fit param check failed for: {0} \n - {1} - """.format(fit_params, str(e))) - if not self.compile_params_list: - plpy.error( "compile_params_list cannot be NULL") - for compile_params in self.compile_params_list: - try: - res = parse_and_validate_compile_params(compile_params) - except Exception as e: - plpy.error( - """Compile param check failed for: {0} \n - {1} - """.format(compile_params, str(e))) - - def _validate_input_output_tables(self): - input_tbl_valid(self.model_arch_table, self.module_name) - output_tbl_valid(self.model_selection_table, self.module_name) - output_tbl_valid(self.model_selection_summary_table, self.module_name) - - diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_wrapper.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_wrapper.py_in deleted file mode 100644 index 541d370..0000000 --- a/src/ports/postgres/modules/deep_learning/madlib_keras_wrapper.py_in +++ /dev/null @@ -1,333 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import ast -import os -import plpy -from math import ceil - -from keras import backend as K -from keras import utils as keras_utils -from keras.optimizers import * - -import keras.optimizers as opt -import keras.losses as losses - -import madlib_keras_serializer -import madlib_keras_gpu_info -from utilities.utilities import _assert -from utilities.utilities import is_platform_pg - -CUDA_VISIBLE_DEVICES_KEY = 'CUDA_VISIBLE_DEVICES' -####################################################################### -########### Keras specific functions ##### -####################################################################### - -def set_cuda_env(value): - """ - :param value: -1 to disable gpu - :return: - """ - os.environ[CUDA_VISIBLE_DEVICES_KEY] = value - -def reset_cuda_env(value): - """ - This function will reset the cuda env variable. This should only be called - if set_cuda_env was called previously. - :param value: - """ - if value: - set_cuda_env(value) - else: - if CUDA_VISIBLE_DEVICES_KEY in os.environ: - del os.environ[CUDA_VISIBLE_DEVICES_KEY] - -def get_device_name_and_set_cuda_env(gpu_count, seg): - - if gpu_count > 0: - device_name = '/gpu:0' - if is_platform_pg(): - cuda_visible_dev = ','.join([str(i) for i in range(gpu_count)]) - else: - cuda_visible_dev = str(seg % gpu_count) - set_cuda_env(cuda_visible_dev) - else: # cpu only - device_name = '/cpu:0' - set_cuda_env('-1') - return device_name - -def set_keras_session(device_name, gpu_count, segments_per_host): - with K.tf.device(device_name): - session = get_keras_session(device_name, gpu_count, segments_per_host) - K.set_session(session) - -def get_keras_session(device_name, gpu_count, segments_per_host): - config = K.tf.ConfigProto() - if gpu_count > 0: - memory_fraction = get_gpu_memory_fraction(gpu_count, segments_per_host) - config.gpu_options.allow_growth = False - config.gpu_options.per_process_gpu_memory_fraction = memory_fraction - session = tf.Session(config=config) - return session - -def clear_keras_session(sess = None): - if sess is None: - sess = K.get_session() - K.clear_session() - sess.close() - - -def get_gpu_memory_fraction(gpu_count, segments_per_host): - """ - We cap the gpu memory usage to 90% of the total available gpu memory. - This 90% is evenly distributed among the segments per gpu. - :param gpu_count: - :param segments_per_host: - :return: - """ - return 0.9 / ceil(1.0 * segments_per_host / gpu_count) - -def get_model_shapes(model): - model_shapes = [] - for a in model.get_weights(): - model_shapes.append(a.shape) - return model_shapes - -def compile_and_set_weights(segment_model, compile_params, device_name, - serialized_weights): - model_shapes = get_model_shapes(segment_model) - compile_model(segment_model, compile_params) - model_weights = madlib_keras_serializer.deserialize_as_nd_weights( - serialized_weights, model_shapes) - segment_model.set_weights(model_weights) - -# TODO: This can be refactored to be part of compile_and_set_weights(), -# by making compile_params an optional param in that function. Doing that -# now might create more merge conflicts with other JIRAs, so get to this later. -def set_model_weights(segment_model, serialized_weights): - model_shapes = get_model_shapes(segment_model) - model_weights = madlib_keras_serializer.deserialize_as_nd_weights( - serialized_weights, model_shapes) - segment_model.set_weights(model_weights) - -""" -Used to convert compile_params and fit_params to actual argument dictionaries -""" - -def convert_string_of_args_to_dict(str_of_args): - """Uses parenthases matching algorithm to intelligently convert - a string with valid python code into an argument dictionary""" - stack = [] - dual = { - '(' : ')', - '[' : ']', - '{' : '}', - } - result_str = "" - key_str = "" - compile_dict = {} - for char in str_of_args: - if char in dual.keys(): - stack.append(char) - result_str += char - elif char in dual.values() and stack: - if dual[stack[-1]] == char: - stack.pop(-1) - result_str += char - elif not stack and char == "=": - key_str = result_str - result_str = "" - elif not stack and char == ",": - value_str = result_str - result_str = "" - compile_dict[key_str.strip()]=value_str.strip().strip('\'') - else: - result_str += char - value_str = result_str - compile_dict[key_str.strip()]=value_str.strip().strip('\'') - return compile_dict - -def get_metrics_from_compile_param(str_of_args): - compile_dict = convert_string_of_args_to_dict(str_of_args) - metrics = None - ckey = 'metrics' - if ckey in compile_dict: - try: - metrics = ast.literal_eval(compile_dict[ckey]) - except ValueError: - plpy.error(("Invalid input value for parameter {0}, " - "please refer to the documentation").format(ckey)) - return metrics - -# Parse the compile parameters and the optimizer. -def parse_and_validate_compile_params(str_of_args): - """ - Args: - @param: str_of_args The string of arguments given by the user - Returns: - opt_name: Name of the optimizer - opt_args: Arguments for the optimizer - compile_dict: Dictionary of arguments for keras.compile - """ - literal_eval_compile_params = ['metrics', 'loss_weights', - 'weighted_metrics', 'sample_weight_mode'] - accepted_compile_params = literal_eval_compile_params + ['optimizer', 'loss'] - - compile_dict = convert_string_of_args_to_dict(str_of_args) - compile_dict = validate_and_literal_eval_keys(compile_dict, - literal_eval_compile_params, - accepted_compile_params) - _assert('optimizer' in compile_dict, "optimizer is a required parameter for compile") - opt_name, opt_args = parse_optimizer(compile_dict) - - _assert('loss' in compile_dict, "loss is a required parameter for compile") - validate_compile_param_types(compile_dict) - _validate_metrics(compile_dict) - return (opt_name, opt_args, compile_dict) - -def _validate_metrics(compile_dict): - _assert('metrics' not in compile_dict.keys() or - compile_dict['metrics'] is None or - type(compile_dict['metrics']) is list, - "wrong input type for compile parameter metrics: multi-output model" - "and user defined metrics are not supported yet, please pass a list") - if 'metrics' in compile_dict and compile_dict['metrics']: - unsupported_metrics_list = ['sparse_categorical_accuracy', - 'sparse_categorical_crossentropy', 'top_k_categorical_accuracy', - 'sparse_top_k_categorical_accuracy'] - _assert(len(compile_dict['metrics']) == 1, - "Only one metric at a time is supported.") - _assert(compile_dict['metrics'][0] not in unsupported_metrics_list, - "Metric {0} is not supported.".format(compile_dict['metrics'][0])) - -# Parse the optimizer name and params. -def parse_optimizer(compile_dict): - """ - Args: - @param: compile_dict Dictionary of arguments for keras.compile - Returns: - opt_name: Name of the optimizer - opt_args: Arguments for the optimizer - """ - opt_split = compile_dict['optimizer'].split('(') - opt_name = opt_split[0] - optimizers = get_optimizers() - _assert(opt_name in optimizers, - "model_keras error: invalid optimizer name: {0}".format(opt_name)) - - # If we use only the optimizer name - if len(opt_split) == 1: - final_args = None - # If we use optimizer object with no params - elif opt_split[1] == ')': - final_args = None - # If we give parameters to the optimizer - else: - opt_params = opt_split[1][:-1] - opt_params_array = opt_params.split(',') - opt_params_clean = map(split_and_strip, opt_params_array) - key_value_params = { x[0] : x[1] for x in opt_params_clean} - - final_args = {} - for key,value in key_value_params.iteritems(): - if value == 'None': - final_args[key] = None - elif value == 'True' or value == 'False': - final_args[key] = bool(value) - else: - final_args[key] = float(value) - return (opt_name,final_args) - - -# Parse the fit parameters into a dictionary. -def parse_and_validate_fit_params(fit_param_str): - - if fit_param_str: - fit_params_dict = convert_string_of_args_to_dict(fit_param_str) - - literal_eval_fit_params = ['batch_size','epochs','verbose', - 'class_weight','initial_epoch','steps_per_epoch'] - accepted_fit_params = literal_eval_fit_params + ['shuffle'] - - fit_params_dict = validate_and_literal_eval_keys(fit_params_dict, - literal_eval_fit_params, - accepted_fit_params) - if 'shuffle' in fit_params_dict: - shuffle_value = fit_params_dict['shuffle'] - if shuffle_value == 'True' or shuffle_value == 'False': - fit_params_dict['shuffle'] = bool(shuffle_value) - - return fit_params_dict - else: - return {} - -# Validate the keys of the given dictionary and run literal_eval on the -# user-defined subset -def validate_and_literal_eval_keys(keys_dict, literal_eval_list, accepted_list): - for ckey in keys_dict.keys(): - _assert(ckey in accepted_list, - "{0} is not currently accepted as a parameter. ".format(ckey)) - if ckey in literal_eval_list: - try: - keys_dict[ckey] = ast.literal_eval(keys_dict[ckey]) - except ValueError: - plpy.error(("invalid input value for parameter {0}, " - "please refer to the documentation").format(ckey)) - return keys_dict - -# Split and strip the whitespace of key=value formatted strings -def split_and_strip(x): - y = x.split('=') - return (y[0].strip(),y[1].strip()) - -# Return the list of keras optimizers -def get_optimizers(): - optimizers = dict() - names = dir(opt) - for n in names: - optimizer = eval('opt.' + n) - if optimizer.__class__ == type and optimizer.__base__ == opt.Optimizer: - optimizers[n] = optimizer - return optimizers - -# Run the keras.compile with the given parameters -def compile_model(model, compile_params): - optimizers = get_optimizers() - (opt_name,final_args,compile_dict) = parse_and_validate_compile_params(compile_params) - compile_dict['optimizer'] = optimizers[opt_name](**final_args) if final_args else opt_name - model.compile(**compile_dict) - -def validate_compile_param_types(compile_dict): - _assert('loss_weights' not in compile_dict.keys() or - compile_dict['loss_weights'] is None or - type(compile_dict['loss_weights']) is list or - type(compile_dict['loss_weights']) is dict, - "wrong input type for compile parameter loss_weights: only list " - "and dictionary are supported.") - - _assert('weighted_metrics' not in compile_dict.keys() or - compile_dict['weighted_metrics'] is None or - type(compile_dict['weighted_metrics']) is list, - "wrong input type for compile parameter weighted_metrics: only list " - "is supported.") - - _assert('sample_weight_mode' not in compile_dict.keys() or - compile_dict['sample_weight_mode'] is None or - compile_dict['sample_weight_mode'] == "temporal", - """compile parameter sample_weight_mode can only be "temporal" or None""") diff --git a/src/ports/postgres/modules/deep_learning/model_arch_info.py_in b/src/ports/postgres/modules/deep_learning/model_arch_info.py_in deleted file mode 100644 index 3a225cc..0000000 --- a/src/ports/postgres/modules/deep_learning/model_arch_info.py_in +++ /dev/null @@ -1,104 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -m4_changequote(`') - -import sys -import json -import plpy -from keras_model_arch_table import ModelArchSchema - -def _get_layers(model_arch): - d = json.loads(model_arch) - config = d['config'] - if type(config) == list: - return config # In keras 2.1.x, all models are sequential - elif type(config) == dict and 'layers' in config: - layers = config['layers'] - if type(layers) == list: - return config['layers'] # In keras 2.x, only sequential models are supported - plpy.error("Unable to read model architecture JSON.") - -def get_input_shape(model_arch): - arch_layers = _get_layers(model_arch) - if 'batch_input_shape' in arch_layers[0]['config']: - return arch_layers[0]['config']['batch_input_shape'][1:] - plpy.error('Unable to get input shape from model architecture.') - -def get_num_classes(model_arch): - """ - We assume that the last dense layer in the model architecture contains the num_classes (units) - An example can be: - ``` - ... - model.add(Flatten()) - model.add(Dense(512)) - model.add(Activation('relu')) - model.add(Dropout(0.5)) - model.add(Dense(num_classes)) - model.add(Activation('softmax')) - ``` - where activation can be after the dense layer. - :param model_arch: - :return: - """ - arch_layers = _get_layers(model_arch) - i = len(arch_layers) - 1 - while i >= 0: - if 'units' in arch_layers[i]['config']: - return arch_layers[i]['config']['units'] - i -= 1 - plpy.error('Unable to get number of classes from model architecture.') - -def get_model_arch_layers_str(model_arch): - arch_layers = _get_layers(model_arch) - layers = "Model arch layers:\n" - first = True - for layer in arch_layers: - if first: - first = False - else: - layers += " |\n" - layers += " V\n" - class_name = layer['class_name'] - config = layer['config'] - if class_name == 'Dense': - layers += "{1}[{2}]\n".format(class_name, config['units']) - else: - layers += "{1}\n".format(class_name) - return layers - -def get_model_arch_weights(model_arch_table, model_id): - - #assume validation is already called - model_arch_query = "SELECT {0}, {1} FROM {2} WHERE {3} = {4}".format( - ModelArchSchema.MODEL_ARCH, ModelArchSchema.MODEL_WEIGHTS, - model_arch_table, ModelArchSchema.MODEL_ID, - model_id) - model_arch_result = plpy.execute(model_arch_query) - if not model_arch_result: - plpy.error("no model arch found in table {0} with id {1}".format( - model_arch_table, model_id)) - - model_arch_result = model_arch_result[0] - - model_arch = model_arch_result[ModelArchSchema.MODEL_ARCH] - model_weights = model_arch_result[ModelArchSchema.MODEL_WEIGHTS] - - return model_arch, model_weights diff --git a/src/ports/postgres/modules/deep_learning/predict_input_params.py_in b/src/ports/postgres/modules/deep_learning/predict_input_params.py_in deleted file mode 100644 index fdab1e4..0000000 --- a/src/ports/postgres/modules/deep_learning/predict_input_params.py_in +++ /dev/null @@ -1,67 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import plpy -from keras_model_arch_table import ModelArchSchema -from utilities.utilities import add_postfix -from utilities.validate_args import input_tbl_valid - -from madlib_keras_helper import CLASS_VALUES_COLNAME -from madlib_keras_helper import DEPENDENT_VARNAME_COLNAME -from madlib_keras_helper import DEPENDENT_VARTYPE_COLNAME -from madlib_keras_helper import MODEL_WEIGHTS_COLNAME -from madlib_keras_helper import NORMALIZING_CONST_COLNAME -from madlib_keras_helper import create_summary_view - -class PredictParamsProcessor: - def __init__(self, model_table, module_name, mst_key): - self.module_name = module_name - self.model_table = model_table - self.mst_key = mst_key - self.model_summary_table = add_postfix(self.model_table, '_summary') - self.mult_where_clause = "" - if self.mst_key: - self.model_summary_table = create_summary_view(self.module_name, - self.model_table, - self.mst_key) - self.mult_where_clause = "WHERE mst_key = {0}".format(self.mst_key) - input_tbl_valid(self.model_summary_table, self.module_name) - self.model_summary_dict = self._get_dict_for_table(self.model_summary_table) - self.model_arch_dict = self._get_dict_for_table(self.model_table) - - def _get_dict_for_table(self, table_name): - return plpy.execute("SELECT * FROM {0} {1}".format(table_name, self.mult_where_clause), 1)[0] - - def get_class_values(self): - return self.model_summary_dict[CLASS_VALUES_COLNAME] - - def get_dependent_varname(self): - return self.model_summary_dict[DEPENDENT_VARNAME_COLNAME] - - def get_dependent_vartype(self): - return self.model_summary_dict[DEPENDENT_VARTYPE_COLNAME] - - def get_model_arch(self): - return self.model_arch_dict[ModelArchSchema.MODEL_ARCH] - - def get_model_weights(self): - return self.model_arch_dict[MODEL_WEIGHTS_COLNAME] - - def get_normalizing_const(self): - return self.model_summary_dict[NORMALIZING_CONST_COLNAME] diff --git a/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in b/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in deleted file mode 100644 index 7c6c5c3..0000000 --- a/src/ports/postgres/modules/deep_learning/test/input_data_preprocessor.sql_in +++ /dev/null @@ -1,645 +0,0 @@ -/* ----------------------------------------------------------------------- *//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ----------------------------------------------------------------------- */ -m4_include(`SQLCommon.m4') -m4_changequote(`') - -DROP TABLE IF EXISTS data_preprocessor_input; -CREATE TABLE data_preprocessor_input(id serial, x double precision[], label TEXT); -INSERT INTO data_preprocessor_input(x, label) VALUES -(ARRAY[1,2,3,4,5,6], 'a'), -(ARRAY[11,2,3,4,5,6], 'a'), -(ARRAY[11,22,33,4,5,6], 'a'), -(ARRAY[11,22,33,44,5,6], 'a'), -(ARRAY[11,22,33,44,65,6], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,144,65,56], 'a'), -(ARRAY[11,22,233,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'b'), -(ARRAY[11,22,33,44,65,56], 'b'), -(ARRAY[11,22,33,44,65,56], 'b'); - -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input', - 'data_preprocessor_input_batch', - 'id', - 'x', - 5); - --- Divide two numbers and round up to the nearest integer -CREATE FUNCTION divide_roundup(numerator NUMERIC, denominator NUMERIC) -RETURNS INTEGER AS -$$ - SELECT (ceil($1 / $2)::INTEGER); -$$ LANGUAGE SQL; - --- num_buffers_calc() represents the num_buffers value that should be --- calculated by the preprocessor. --- For postgres, just need total rows / buffer_size rounded up. --- For greenplum, we take that result, and round up to the nearest multiple --- of num_segments. -CREATE FUNCTION num_buffers_calc(rows_in_tbl INTEGER, buffer_size INTEGER) -RETURNS INTEGER AS -$$ -m4_ifdef(, - , - -) -$$ LANGUAGE SQL; - --- num_buffers() represents the actual number of buffers expected to --- be returned in the output table. --- For postgres, this should always be the same as num_buffers_calc() --- (as long as rows_in_tbl > 0, which should be validated elsewhere) --- For greenplum, this can be less than num_buffers_calc() in --- the special case where there is only one row per buffer. In --- that case, the buffers in the output table will be equal to --- the number of rows in the input table. This can only happen --- if rows_in_tbl < num_segments and is the only case where the --- number of buffers on each segment will not be exactly equal -CREATE FUNCTION num_buffers(rows_in_tbl INTEGER, buffer_size INTEGER) -RETURNS INTEGER AS -$$ - SELECT LEAST(num_buffers_calc($1, $2), $1); -$$ LANGUAGE SQL; - -CREATE FUNCTION buffer_size(rows_in_tbl INTEGER, requested_buffer_size INTEGER) -RETURNS INTEGER AS -$$ - SELECT divide_roundup($1, num_buffers($1, $2)); -$$ LANGUAGE SQL; - -SELECT assert(COUNT(*) = num_buffers(17, 5), - 'Incorrect number of buffers in data_preprocessor_input_batch.') -FROM data_preprocessor_input_batch; - -SELECT assert(independent_var_shape[2]=6, 'Incorrect image shape ' || independent_var_shape[2]) -FROM data_preprocessor_input_batch WHERE buffer_id=0; - -SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1]) -FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; - -SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1]) -FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=1; - -SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size ' || independent_var_shape[1]) -FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=2; - -SELECT assert(total_images = 17, 'Incorrect total number of images! Last buffer has incorrect size?') -FROM (SELECT SUM(independent_var_shape[1]) AS total_images FROM data_preprocessor_input_batch) a; - -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length ' || octet_length(independent_var)::TEXT) -FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; - - -DROP TABLE IF EXISTS validation_out, validation_out_summary; -SELECT validation_preprocessor_dl( - 'data_preprocessor_input', - 'validation_out', - 'id', - 'x', - 'data_preprocessor_input_batch', - 5); - -SELECT assert(COUNT(*) = num_buffers(17, 5), - 'Incorrect number of buffers in validation_out.') -FROM validation_out; - -SELECT assert(independent_var_shape[2]=6, 'Incorrect image shape.') -FROM data_preprocessor_input_batch WHERE buffer_id=0; - -SELECT assert(independent_var_shape[1]=buffer_size, 'Incorrect buffer size.') -FROM (SELECT buffer_size(17, 5) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=1; - -SELECT assert(total_images = 17, 'Incorrect total number of images! Last buffer has incorrect size?') -FROM (SELECT SUM(independent_var_shape[1]) AS total_images FROM data_preprocessor_input_batch) a; - -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length') -FROM (SELECT buffer_size(17, 5) buffer_size) a, validation_out WHERE buffer_id=0; - -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input', - 'data_preprocessor_input_batch', - 'label', - 'x'); - --- Test data is evenly distributed across all segments (GPDB only) -m4_ifdef(, , ) - -DROP TABLE IF EXISTS data_preprocessor_input; -CREATE TABLE data_preprocessor_input(id serial, x double precision[], y INTEGER, y1 BOOLEAN, y2 TEXT, y3 DOUBLE PRECISION, y4 DOUBLE PRECISION[], y5 INTEGER[]); -INSERT INTO data_preprocessor_input(x, y, y1, y2, y3, y4, y5) VALUES -(ARRAY[1,2,3,4,5,6], 4, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,2,3,4,5,6], 3, TRUE, 'c', 4.2, ARRAY[0.0, 1.0], ARRAY[1,0]), -(ARRAY[11,22,33,4,5,6], 8, TRUE, 'a', 4.0, ARRAY[0.0, 1.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,5,6], 2, FALSE, 'a', 4.2, ARRAY[0.0, 1.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,6], 5, TRUE, 'b', 4.0, ARRAY[0.0, 1.0], ARRAY[0,1]), -(ARRAY[11,22,33,44,65,56], 6, TRUE, 'a', 5.0, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,56], 2, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,56], 10, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,56], 3, TRUE, 'b', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,56], 7, FALSE, 'a', 5.0, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,56], 6, TRUE, 'a', 4.0, ARRAY[0.0, 1.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,56], -6, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,22,33,144,65,56], 9, TRUE, 'c', 4.0, ARRAY[0.0, 1.0], ARRAY[1,0]), -(ARRAY[11,22,233,44,65,56], 0, TRUE, 'a', 5.0, ARRAY[1.0, 0.0], ARRAY[0,1]), -(ARRAY[11,22,33,44,65,56], 12, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,56], -3, FALSE, 'a', 4.2, ARRAY[1.0, 0.0], ARRAY[1,0]), -(ARRAY[11,22,33,44,65,56], -1, TRUE, 'b', 4.0, ARRAY[0.0, 1.0], ARRAY[0,1]); - -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input', - 'data_preprocessor_input_batch', - 'y', - 'x', - 4, - 5, - 16 -- num_classes - ); - --- Test that indepdendent vars get divided by 5, by verifying min value goes from 1 to 0.2, and max value from 233 to 46.6 -SELECT assert(relative_error(MIN(x),0.2) < 0.00001, 'Independent var not normalized properly!') FROM (SELECT UNNEST(convert_bytea_to_real_array(independent_var)) as x FROM data_preprocessor_input_batch) a; -SELECT assert(relative_error(MAX(x),46.6) < 0.00001, 'Independent var not normalized properly!') FROM (SELECT UNNEST(convert_bytea_to_real_array(independent_var)) as x FROM data_preprocessor_input_batch) a; --- Test that 1-hot encoded array is of length 16 (num_classes) -SELECT assert(dependent_var_shape[2] = 16, 'Incorrect one-hot encode dimension with num_classes') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; - -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length') -FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; - --- Test summary table -SELECT assert - ( - source_table = 'data_preprocessor_input' AND - output_table = 'data_preprocessor_input_batch' AND - dependent_varname = 'y' AND - independent_varname = 'x' AND - dependent_vartype = 'integer' AND - class_values = '{-6,-3,-1,0,2,3,4,5,6,7,8,9,10,12,NULL,NULL}' AND - summary.buffer_size = a.buffer_size AND -- we sort the class values in python - normalizing_const = 5 AND - pg_typeof(normalizing_const) = 'real'::regtype AND - num_classes = 16 AND - distribution_rules = 'all_segments', - 'Summary Validation failed. Actual:' || __to_char(summary) - ) FROM (SELECT buffer_size(17, 4) buffer_size) a, - (SELECT * FROM data_preprocessor_input_batch_summary) summary; - ---- Test output data type -SELECT assert(pg_typeof(independent_var) = 'bytea'::regtype, 'Wrong independent_var type') FROM data_preprocessor_input_batch WHERE buffer_id = 0; -SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'Wrong dependent_var type') FROM data_preprocessor_input_batch WHERE buffer_id = 0; - --- Test for validation data where the input table has only a subset of --- the classes compared to the original training data --- (data_preprocessor_input_batch). The one hot encoding must be based --- on class_values from data_preprocessor_input_batch_summary rather --- than the class levels found in validation_input. -DROP TABLE IF EXISTS validation_input; -CREATE TABLE validation_input(id serial, x_new double precision[], y_new INTEGER, y1 BOOLEAN, y2 TEXT, y3 DOUBLE PRECISION, y4 DOUBLE PRECISION[], y5 INTEGER[]); -INSERT INTO validation_input(x_new, y_new, y1, y2, y3, y4, y5) VALUES -(ARRAY[1,2,3,4,5,6], 4, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]); -DROP TABLE IF EXISTS validation_out, validation_out_summary; -SELECT validation_preprocessor_dl( - 'validation_input', - 'validation_out', - 'y_new', - 'x_new', - 'data_preprocessor_input_batch'); --- Hard code 5.0 as the normalizing constant, based on the previous --- query's input param, to test if normalization is correct. -SELECT assert(abs(x_new[1]/5.0-(convert_bytea_to_real_array(independent_var))[1]) < 0.0000001, 'Incorrect normalizing in validation table.') -FROM validation_input, validation_out; --- Validate if one hot encoding is as expected. -SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0}', 'Incorrect one-hot encode dimension with num_classes') FROM - validation_out WHERE buffer_id = 0; - --- Test summary table -SELECT assert - ( - source_table = 'validation_input' AND - output_table = 'validation_out' AND - dependent_varname = 'y_new' AND - independent_varname = 'x_new' AND - dependent_vartype = 'integer' AND - class_values = '{-6,-3,-1,0,2,3,4,5,6,7,8,9,10,12,NULL,NULL}' AND - buffer_size = 1 AND -- we sort the class values in python - normalizing_const = 5 AND - pg_typeof(normalizing_const) = 'real'::regtype AND - num_classes = 16, - 'Summary Validation failed. Actual:' || __to_char(summary) - ) from (select * from validation_out_summary) summary; - --- Test one-hot encoding for dependent_var --- test boolean type -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input', - 'data_preprocessor_input_batch', - 'y1', - 'x', - 4, - 5); -SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0; -SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; - -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length') -FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; - -SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:2]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0; -SELECT assert (dependent_vartype = 'boolean' AND - class_values = '{f,t}' AND - num_classes = 2, - 'Summary Validation failed. Actual:' || __to_char(summary) - ) from (select * from data_preprocessor_input_batch_summary) summary; - --- Test to assert the output summary table for validation has the correct --- num_classes and class_values -DROP TABLE IF EXISTS validation_input; -CREATE TABLE validation_input(id serial, x_new double precision[], y INTEGER, y_new BOOLEAN, y2 TEXT, y3 DOUBLE PRECISION, y4 DOUBLE PRECISION[], y5 INTEGER[]); -INSERT INTO validation_input(x_new, y, y_new, y2, y3, y4, y5) VALUES -(ARRAY[1,2,3,4,5,6], 4, TRUE, 'a', 4.0, ARRAY[1.0, 0.0], ARRAY[1,0]); -DROP TABLE IF EXISTS validation_out, validation_out_summary; -SELECT validation_preprocessor_dl( - 'validation_input', - 'validation_out', - 'y_new', - 'x_new', - 'data_preprocessor_input_batch'); - -SELECT assert (dependent_vartype = 'boolean' AND - class_values = '{f,t}' AND - num_classes = 2, - 'Summary Validation failed. Actual:' || __to_char(summary) - ) from (select * from validation_out_summary) summary; --- test text type -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input', - 'data_preprocessor_input_batch', - 'y2', - 'x', - 4, - 5); -SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0; -SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; - -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length') -FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; - -SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:3]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0; -SELECT assert (dependent_vartype = 'text' AND - class_values = '{a,b,c}' AND - num_classes = 3, - 'Summary Validation failed. Actual:' || __to_char(summary) - ) from (select * from data_preprocessor_input_batch_summary) summary; - --- data_preprocessor_input_batch was created for boolean dependent var, --- trying to create validation data using this table for reference, but --- passing a non-boolean dep var should error out. -DROP TABLE IF EXISTS validation_out, validation_out_summary; -select assert(trap_error($TRAP$SELECT validation_preprocessor_dl( - 'validation_input', - 'validation_out', - 'y_new', - 'x_new', - 'data_preprocessor_input_batch');$TRAP$) = 1, - 'Passing boolean dep var while expecting text dep var should error out.'); - - --- test double precision type -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input', - 'data_preprocessor_input_batch', - 'y3', - 'x', - 4, - 5); -SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0; -SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length') -FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; -SELECT assert(SUM(y) = 1, 'Incorrect one-hot encode format') FROM (SELECT buffer_id, UNNEST((convert_bytea_to_smallint_array(dependent_var))[1:3]) as y FROM data_preprocessor_input_batch) a WHERE buffer_id = 0; -SELECT assert (dependent_vartype = 'double precision' AND - class_values = '{4.0,4.2,5.0}' AND - num_classes = 3, - 'Summary Validation failed. Actual:' || __to_char(summary) - ) from (select * from data_preprocessor_input_batch_summary) summary; - --- test double precision array type -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input', - 'data_preprocessor_input_batch', - 'y4', - 'x', - 4, - 5); -SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0; -SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; - -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length') -FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; - -SELECT assert(relative_error(SUM(y), SUM(y4)) < 0.000001, 'Incorrect one-hot encode value') FROM (SELECT UNNEST(convert_bytea_to_smallint_array(dependent_var)) AS y FROM data_preprocessor_input_batch) a, (SELECT UNNEST(y4) as y4 FROM data_preprocessor_input) b; -SELECT assert (dependent_vartype = 'double precision[]' AND - class_values IS NULL AND - num_classes IS NULL, - 'Summary Validation failed. Actual:' || __to_char(summary) - ) from (select * from data_preprocessor_input_batch_summary) summary; - -DROP TABLE IF EXISTS validation_out, validation_out_summary; -SELECT validation_preprocessor_dl( - 'validation_input', - 'validation_out', - 'y4', - 'x_new', - 'data_preprocessor_input_batch'); - -SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0}' AND dependent_var_shape[2] = 2, 'Incorrect one-hot encoding for already encoded dep var') FROM - validation_out WHERE buffer_id = 0; - --- test integer array type -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input', - 'data_preprocessor_input_batch', - 'y5', - 'x', - 4, - 5); -SELECT assert(pg_typeof(dependent_var) = 'bytea'::regtype, 'One-hot encode doesn''t convert into integer array format') FROM data_preprocessor_input_batch WHERE buffer_id = 0; -SELECT assert(dependent_var_shape[2] = 2, 'Incorrect one-hot encode dimension') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; - -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length') -FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; - -SELECT assert(relative_error(SUM(y), SUM(y5)) < 0.000001, 'Incorrect one-hot encode value') FROM (SELECT UNNEST(convert_bytea_to_smallint_array(dependent_var)) AS y FROM data_preprocessor_input_batch) a, (SELECT UNNEST(y5) as y5 FROM data_preprocessor_input) b; -SELECT assert (dependent_vartype = 'integer[]' AND - class_values IS NULL AND - num_classes IS NULL, - 'Summary Validation failed. Actual:' || __to_char(summary) - ) from (select * from data_preprocessor_input_batch_summary) summary; - --- Test cases with NULL in class values -DROP TABLE IF EXISTS data_preprocessor_input_null; -CREATE TABLE data_preprocessor_input_null(id serial, x double precision[], label TEXT); -INSERT INTO data_preprocessor_input_null(x, label) VALUES -(ARRAY[1,2,3,4,5,6], 'a'), -(ARRAY[11,2,3,4,5,6], 'a'), -(ARRAY[11,22,33,4,5,6], NULL), -(ARRAY[11,22,33,44,5,6], 'a'), -(ARRAY[11,22,33,44,65,6], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], NULL), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,44,65,56], NULL), -(ARRAY[11,22,33,44,65,56], 'a'), -(ARRAY[11,22,33,144,65,56], 'b'), -(ARRAY[11,22,233,44,65,56], 'b'), -(ARRAY[11,22,33,44,65,56], 'b'), -(ARRAY[11,22,33,44,65,56], 'b'), -(ARRAY[11,22,33,44,65,56], NULL); - -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input_null', - 'data_preprocessor_input_batch', - 'label', - 'x', - 4, - 5, - 5 -- num_classes - ); --- Test summary table if class_values has NULL as a legitimate --- class label, and also two other NULLs because num_classes=5 --- but table has only 3 distinct class labels (including NULL) -SELECT assert - ( - class_values = '{NULL,a,b,NULL,NULL}', - 'Summary Validation failed with NULL data. Actual:' || __to_char(summary) - ) from (select * from data_preprocessor_input_batch_summary) summary; - -SELECT assert(dependent_var_shape[2] = 5, 'Incorrect one-hot encode dimension') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; - -SELECT assert(octet_length(independent_var) = buffer_size*6*4, 'Incorrect buffer length') -FROM (SELECT buffer_size(17, 4) buffer_size) a, data_preprocessor_input_batch WHERE buffer_id=0; - --- The same tests, but for validation. -DROP TABLE IF EXISTS data_preprocessor_input_validation_null; -CREATE TABLE data_preprocessor_input_validation_null(id serial, x double precision[], label TEXT); -INSERT INTO data_preprocessor_input_validation_null(x, label) VALUES -(ARRAY[1,2,3,4,5,6], 'a'), -(ARRAY[11,2,3,4,5,6], 'b'), -(ARRAY[11,2,3,4,5,6], 'b'), -(ARRAY[111,22,33,4,5,6], NULL); -DROP TABLE IF EXISTS validation_out_batch, validation_out_batch_summary; -SELECT validation_preprocessor_dl( - 'data_preprocessor_input_validation_null', - 'validation_out_batch', - 'label', - 'x', - 'data_preprocessor_input_batch', - 1 - ); - --- Test summary table if class_values has NULL as a legitimate --- class label, and also two other NULLs because num_classes=5 --- but table has only 3 distinct class labels (including NULL) -SELECT assert - ( - class_values = '{NULL,a,b,NULL,NULL}', - 'Summary Validation failed with NULL data. Actual:' || __to_char(summary) - ) from (select * from validation_out_batch_summary) summary; - --- Validate one hot encoding for specific row is correct -SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{0,1,0,0,0}' AND dependent_var_shape[2] =5, 'Incorrect normalizing in validation table.') -FROM data_preprocessor_input_validation_null, validation_out_batch -WHERE x[1]=1 AND abs((convert_bytea_to_real_array(independent_var))[1] - 0.2::REAL) < 0.00001; --- Assert one-hot encoding for NULL label -SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0,0,0}' AND dependent_var_shape[2] =5, 'Incorrect normalizing in validation table.') -FROM data_preprocessor_input_validation_null, validation_out_batch -WHERE x[1]=111 AND abs((convert_bytea_to_real_array(independent_var))[1] - 22.2::REAL) < 0.00001; - --- Test the content of 1-hot encoded dep var when NULL is the --- class label. -DROP TABLE IF EXISTS data_preprocessor_input_null; -CREATE TABLE data_preprocessor_input_null(id serial, x double precision[], label TEXT); -INSERT INTO data_preprocessor_input_null(x, label) VALUES -(ARRAY[11,22,33,4,5,6], NULL); - -DROP TABLE IF EXISTS data_preprocessor_input_batch, data_preprocessor_input_batch_summary; -SELECT training_preprocessor_dl( - 'data_preprocessor_input_null', - 'data_preprocessor_input_batch', - 'label', - 'x', - 4, - 5, - 3 -- num_classes - ); - --- class_values must be '{NULL,NULL,NULL}' where the first NULL --- is for the class label seen in data, and the other two NULLs --- are added as num_classes=3. -SELECT assert - ( - class_values = '{NULL,NULL,NULL}', - 'Summary Validation failed with NULL data. Actual:' || __to_char(summary) - ) from (select * from data_preprocessor_input_batch_summary) summary; - -SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; -SELECT assert(octet_length(independent_var) = 24, 'Incorrect buffer length') -FROM data_preprocessor_input_batch WHERE buffer_id=0; --- NULL is treated as a class label, so it should show '1' for the --- first index -SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0}', 'Incorrect one-hot encode dimension with NULL data') FROM - data_preprocessor_input_batch WHERE buffer_id = 0; - --- The same tests for validation. -DROP TABLE IF EXISTS validation_out_batch, validation_out_batch_summary; -SELECT validation_preprocessor_dl( - 'data_preprocessor_input_null', - 'validation_out_batch', - 'label', - 'x', - 'data_preprocessor_input_batch', - 4 - ); - --- class_values must be '{NULL,NULL,NULL}' where the first NULL --- is for the class label seen in data, and the other two NULLs --- are added as num_classes=3. -SELECT assert - ( - class_values = '{NULL,NULL,NULL}', - 'Summary Validation failed with NULL data. Actual:' || __to_char(summary) - ) from (select * from validation_out_batch_summary) summary; - -SELECT assert(dependent_var_shape[2] = 3, 'Incorrect one-hot encode dimension') FROM - validation_out_batch WHERE buffer_id = 0; -SELECT assert(octet_length(independent_var) = 24, 'Incorrect buffer length') -FROM data_preprocessor_input_batch WHERE buffer_id=0; --- NULL is treated as a class label, so it should show '1' for the --- first index -SELECT assert(convert_bytea_to_smallint_array(dependent_var) = '{1,0,0}', 'Incorrect one-hot encode dimension with NULL data') FROM - validation_out_batch WHERE buffer_id = 0; - --- Test if validation class values is not a subset of training data class values. -DROP TABLE IF EXISTS data_preprocessor_input_validation_null; -CREATE TABLE data_preprocessor_input_validation_null(id serial, x double precision[], label TEXT); -INSERT INTO data_preprocessor_input_validation_null(x, label) VALUES -(ARRAY[11,2,3,4,5,6], 'bbbb'); -DROP TABLE IF EXISTS validation_out_batch, validation_out_batch_summary; -select assert(trap_error($TRAP$SELECT validation_preprocessor_dl( - 'data_preprocessor_input_validation_null', - 'validation_out_batch', - 'label', - 'x', - 'data_preprocessor_input_batch', - 1);$TRAP$) = 1, - 'validation class values should be a subset of training data class values.'); diff --git a/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.ic.sql_in b/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.ic.sql_in deleted file mode 100644 index 8d82a43..0000000 --- a/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.ic.sql_in +++ /dev/null @@ -1,31 +0,0 @@ -/* ----------------------------------------------------------------------- - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * ----------------------------------------------------------------------- */ - -/* ----------------------------------------------------------------------------- - * Test Keras Model Arch Table helper functions - * -------------------------------------------------------------------------- */ - -SELECT load_keras_model('test239587_keras_model_arch_table','{"a" : 1, "b" : 2, "c" : [4,5,6] }'); -SELECT load_keras_model('test239587_keras_model_arch_table','{"config" : [1,2,3]}'); -SELECT load_keras_model('test239587_keras_model_arch_table','{"config" : [8,4,0]}'); -SELECT delete_keras_model('test239587_keras_model_arch_table',2); -SELECT delete_keras_model('test239587_keras_model_arch_table',3); -SELECT delete_keras_model('test239587_keras_model_arch_table',1); diff --git a/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.sql_in b/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.sql_in deleted file mode 100644 index 1f2009b..0000000 --- a/src/ports/postgres/modules/deep_learning/test/keras_model_arch_table.sql_in +++ /dev/null @@ -1,154 +0,0 @@ -/* ----------------------------------------------------------------------- - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * ----------------------------------------------------------------------- */ - -/* ----------------------------------------------------------------------------- - * Test Keras Model Arch Table helper functions - * -------------------------------------------------------------------------- */ - - -/* Test successful model creation where no table exists */ -DROP TABLE IF EXISTS test_keras_model_arch_table; -SELECT load_keras_model('test_keras_model_arch_table', '{"a" : 1, "b" : 2, "c" : [4,5,6] }'); - -SELECT assert(UPPER(atttypid::regtype::TEXT) = 'INTEGER', 'model_id column should be INTEGER type') - FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass - AND attname = 'model_id'; -SELECT assert(UPPER(atttypid::regtype::TEXT) = 'JSON', 'model_arch column should be JSON type' ) FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass - AND attname = 'model_arch'; -SELECT assert(UPPER(atttypid::regtype::TEXT) = 'BYTEA', - 'model_weights column should be bytea type') - FROM pg_attribute WHERE attrelid = 'test_keras_model_arch_table'::regclass - AND attname = 'model_weights'; - -/* model id should be 1 */ -SELECT assert(model_id = 1, 'Wrong model_id written by load_keras_model') - FROM test_keras_model_arch_table; - -/* model arch should be valid json, with all fields accessible with json operators */ -SELECT assert((model_arch->>'a') = '1', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table; -SELECT assert((model_arch->>'b') = '2', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table; -SELECT assert((model_arch->'c')->>0 = '4', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table; -SELECT assert((model_arch->'c')->>1 = '5', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table; -SELECT assert((model_arch->'c')->>2 = '6', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table; -/* model_weights should be set to null, since this is not a warm start */ -SELECT assert(model_weights IS NULL, 'model_weights should be NULL after load_keras_model() called.') FROM test_keras_model_arch_table; - - -/* Test model creation where valid table exists */ -SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}'); -SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [8,4,0]}'); -SELECT assert(model_arch->'config'->>0 = '1', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table WHERE model_id = 2; -SELECT assert(model_arch->'config'->>1 = '2', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table WHERE model_id = 2; -SELECT assert(model_arch->'config'->>2 = '3', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table WHERE model_id = 2; -SELECT assert(model_arch->'config'->>0 = '8', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table WHERE model_id = 3; -SELECT assert(model_arch->'config'->>1 = '4', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table WHERE model_id = 3; -SELECT assert(model_arch->'config'->>2 = '0', 'Cannot parse model_arch json in model table.') - FROM test_keras_model_arch_table WHERE model_id = 3; - -/* Test deletion where valid table exists */ -SELECT delete_keras_model('test_keras_model_arch_table', 2); -SELECT assert(COUNT(model_id) = 0, 'model id 2 should have been deleted!') - FROM test_keras_model_arch_table WHERE model_id = 2; -SELECT delete_keras_model('test_keras_model_arch_table', 3); -SELECT assert(COUNT(model_id) = 0, 'model id 3 should have been deleted!') - FROM test_keras_model_arch_table WHERE model_id = 3; - /* Delete a second time, to make sure nothing weird happens. - * It should archrt to the user that the model_id wasn't found but not - * raise an exception or change anything. */ -SELECT delete_keras_model('test_keras_model_arch_table', 1); -SELECT assert(trap_error($$SELECT * from test_keras_model_arch_table$$) = 1, - 'Table test_keras_model_arch_table should have been deleted.'); - -SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}'); -DELETE FROM test_keras_model_arch_table; - -/* Test deletion where invalid table exists */ -SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}'); -ALTER TABLE test_keras_model_arch_table DROP COLUMN model_id; - -/* Test deletion where empty table exists */ -select assert(trap_error($$SELECT delete_keras_model('test_keras_model_arch_table', 3)$$) = 1, - 'Deleting a model in an empty table should generate an exception.'); - -SELECT assert(trap_error($$SELECT delete_keras_model('test_keras_model_arch_table', 1)$$) = 1, - 'Deleting an invalid table should generate an exception.'); - -SELECT assert(trap_error($$SELECT load_keras_model('test_keras_model_arch_table', '{"config" : 1}')$$) = 1, 'Passing an invalid table to load_keras_model() should raise exception.'); - -/* Test deletion where no table exists */ -DROP TABLE IF EXISTS test_keras_model_arch_table; -SELECT assert(trap_error($$SELECT delete_keras_model('test_keras_model_arch_table', 3)$$) = 1, - 'Deleting a non-existent table should raise exception.'); - -DROP TABLE IF EXISTS test_keras_model_arch_table; -SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}', 'dummy weights'::bytea); -SELECT load_keras_model('test_keras_model_arch_table', '{"config" : [1,2,3]}', NULL, 'my name', 'my desc'); - -/* Test model weights */ -SELECT assert(model_weights = 'dummy weights', 'Incorrect model_weights in the model arch table.') -FROM test_keras_model_arch_table WHERE model_id = 1; -SELECT assert(model_weights IS NULL, 'model_weights is not NULL') -FROM test_keras_model_arch_table WHERE model_id = 2; - -/* Test name and description */ -SELECT assert(name IS NULL AND description IS NULL, 'Name or description is not NULL.') -FROM test_keras_model_arch_table WHERE model_id = 1; -SELECT assert(name = 'my name' AND description = 'my desc', 'Incorrect name or description in the model arch table.') -FROM test_keras_model_arch_table WHERE model_id = 2; - - ---------------------------- Test calling the UDF from python --------------------------------- -CREATE OR REPLACE FUNCTION create_model_arch_transfer_learning() RETURNS VOID AS $$ -from keras.layers import * -from keras import Sequential -import numpy as np -import plpy - -model = Sequential() -model.add(Conv2D(1, kernel_size=(1, 1), activation='relu', input_shape=(1,1,1,))) -weights = model.get_weights() -weights_flat = [ w.flatten() for w in weights ] -weights1d = np.array([j for sub in weights_flat for j in sub]) -weights1d = np.ones_like(weights1d) -weights_bytea = weights1d.tostring() - -load_query = plpy.prepare("""SELECT load_keras_model( - 'test_keras_model_arch_table', - $1, $2) - """, ['json','bytea']) -plpy.execute(load_query, [model.to_json(), weights_bytea]) -$$ LANGUAGE plpythonu VOLATILE; - -DROP TABLE IF EXISTS test_keras_model_arch_table; -SELECT create_model_arch_transfer_learning(); - -select assert(model_weights = '\000\000\200?\000\000\200?', 'loading weights from udf failed') -from test_keras_model_arch_table; diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in deleted file mode 100644 index 7c9ad5e..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_cifar.setup.sql_in +++ /dev/null @@ -1,118 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - --------------------- CIFAR 10 test input tables ----------------- - -DROP TABLE IF EXISTS cifar_10_sample; -CREATE TABLE cifar_10_sample(id INTEGER, y SMALLINT, y_text TEXT, imgpath TEXT, x REAL[]); -COPY cifar_10_sample FROM STDIN DELIMITER '|'; -0|0|'cat'|'0/img0.jpg'|{{{202,204,199},{202,204,199},{204,206,201},{206,208,203},{208,210,205},{209,211,206},{210,212,207},{212,214,210},{213,215,212},{215,217,214},{216,218,215},{216,218,215},{215,217,214},{216,218,215},{216,218,215},{216,218,214},{217,219,214},{217,219,214},{218,220,215},{218,219,214},{216,217,212},{217,218,213},{218,219,214},{214,215,209},{213,214,207},{212,213,206},{211,212,205},{209,210,203},{208,209,202},{207,208,200},{205,206,199},{203,204,198}},{{206,208,203},{206,208,203},{207,209,204},{209,211,207},{211,213,209},{212,214,210},{213,215,211},{215,217,214},{216,218,215},{218,220,217},{220,222,219},{220,222,219},{220,222,219},{220,222,220},{221,223,219},{220,222,218},{220,222,217},{221,223,218},{221,223,218},{220,222,217},{218,219,214},{215,216,211},{220,221,216},{220,221,215},{217,218,212},{215,216,210},{214,215,209},{213,214,208},{212,213,207},{210,211,205},{208,209,203},{206,207,201}},{{210,212,207},{211,213,208},{212,214,210},{214,216,213},{215,217,215},{217,219,216},{218,220,217},{219,221,219},{221,223,220},{221,223,220},{222,224,221},{223,225,222},{224,226,223},{226,228,225},{226,228,225},{226,228,224},{226,228,223},{227,229,224},{227,228,223},{226,228,223},{225,226,221},{221,222,217},{221,222,217},{223,224,219},{221,222,218},{219,220,216},{219,220,216},{218,219,214},{216,216,212},{214,214,210},{212,213,208},{210,211,206}},{{212,214,209},{213,215,210},{215,217,213},{217,219,217},{218,220,219},{220,222,221},{222,224,223},{224,226,223},{225,227,224},{226,228,225},{230,232,229},{230,232,229},{228,230,227},{229,231,228},{230,232,229},{231,233,229},{232,234,229},{233,235,230},{233,235,230},{233,235,229},{234,235,230},{232,233,228},{229,230,225},{228,229,226},{228,228,226},{226,226,224},{224,224,222},{222,222,220},{220,220,218},{218,218,216},{216,216,213},{213,214,209}},{{215,217,212},{216,218,213},{218,220,215},{219,221,219},{221,223,222},{222,224,223},{224,226,225},{225,227,224},{227,229,225},{224,226,223},{209,211,209},{223,224,222},{232,234,231},{232,234,231},{234,236,233},{235,237,232},{237,239,234},{237,239,235},{238,240,235},{240,241,237},{240,241,236},{240,241,236},{240,241,236},{238,239,236},{237,237,235},{236,236,234},{234,234,232},{231,231,229},{228,229,227},{225,225,223},{221,221,219},{218,219,215}},{{218,220,217},{218,220,216},{220,222,218},{221,223,221},{223,225,222},{225,227,224},{226,228,225},{226,228,223},{232,235,228},{183,185,183},{79,80,84},{155,157,160},{236,238,235},{235,237,232},{236,238,236},{239,241,237},{241,243,240},{244,245,244},{244,246,245},{241,245,243},{242,245,242},{243,245,240},{245,246,241},{246,246,242},{245,246,241},{245,246,242},{244,245,241},{243,244,241},{240,241,239},{236,238,235},{232,234,231},{228,230,228}},{{221,223,220},{221,223,220},{222,224,221},{224,226,223},{226,228,224},{230,232,229},{232,234,231},{231,233,229},{235,238,232},{170,172,171},{34,34,42},{92,92,101},{223,224,224},{238,240,234},{238,240,236},{235,237,234},{216,218,216},{194,195,196},{200,202,205},{237,242,242},{244,247,245},{245,246,242},{247,248,242},{248,249,244},{249,250,245},{250,251,246},{250,251,246},{250,252,248},{249,251,248},{247,249,246},{246,248,245},{243,245,243}},{{222,224,221},{223,225,222},{225,227,224},{227,229,226},{230,232,229},{187,189,186},{172,174,171},{227,229,228},{243,245,241},{179,181,180},{38,38,46},{47,47,60},{191,192,197},{240,243,237},{242,245,238},{186,188,185},{82,84,83},{63,64,67},{72,73,77},{184,188,189},{246,250,249},{245,246,244},{247,248,244},{249,250,245},{249,250,245},{250,251,246},{251,253,248},{252,254,251},{252,254,251},{252,254,251},{252,254,252},{252,254,253}},{{222,224,221},{223,225,222},{226,228,225},{229,231,228},{233,235,232},{158,161,157},{72,74,72},{107,108,110},{190,192,191},{183,184,184},{44,45,52},{26,25,39},{135,135,141},{241,243,239},{246,248,243},{152,154,152},{37,38,39},{46,47,51},{47,48,53},{96,100,103},{223,226,228},{248,249,248},{247,247,245},{249,250,245},{249,250,245},{250,251,246},{250,251,246},{250,252,249},{250,252,250},{251,253,250},{251,253,250},{251,253,252}},{{222,225,221},{222,224,221},{225,228,224},{229,231,228},{232,234,231},{220,222,219},{163,165,163},{81,81,86},{70,71,74},{98,100,100},{41,41,48},{23,23,35},{76,77,83},{202,204,203},{221,223,223},{124,125,126},{42,43,46},{52,53,58},{51,52,59},{51,54,59},{158,161,164},{248,249,250},{248,248,248},{249,250,246},{249,250,245},{250,251,246},{250,251,246},{250,252,248},{250,252,249},{250,252,249},{250,252,249},{250,252,251}},{{223,225,223},{223,225,223},{227,228,226},{230,232,229},{231,233,231},{234,236,233},{236,238,236},{195,196,200},{103,104,107},{38,39,40},{25,26,30},{32,33,41},{39,41,45},{81,82,85},{93,95,99},{70,71,74},{46,47,51},{48,49,54},{48,50,56},{42,46,51},{81,84,87},{219,220,222},{251,251,251},{249,250,245},{249,250,245},{250,251,246},{250,251,246},{250,252,248},{250,252,249},{249,251,248},{249,251,248},{249,251,250}},{{224,224,225},{225,225,226},{228,228,229},{230,231,230},{232,234,233},{233,235,233},{234,236,235},{235,237,234},{212,214,211},{125,127,125},{24,26,26},{16,18,21},{28,30,34},{35,37,41},{48,50,54},{50,52,56},{56,59,62},{52,55,59},{43,46,50},{47,53,58},{47,53,57},{145,148,149},{246,247,246},{250,251,246},{251,252,246},{251,252,247},{251,252,247},{251,252,247},{251,252,247},{250,251,247},{250,251,247},{250,251,249}},{{224,224,226},{225,225,227},{228,228,230},{230,231,231},{232,234,233},{234,235,235},{235,237,235},{235,237,232},{241,243,238},{148,150,147},{19,21,21},{12,13,17},{12,13,17},{19,20,24},{29,30,34},{28,29,33},{51,52,56},{53,54,58},{35,37,41},{40,46,52},{57,63,67},{70,73,76},{168,168,170},{247,248,244},{251,252,247},{251,252,247},{251,252,247},{251,252,247},{252,253,248},{251,252,247},{251,252,247},{251,251,249}},{{224,224,226},{225,225,227},{228,228,230},{229,231,231},{232,234,232},{233,235,234},{235,237,236},{238,240,236},{226,228,224},{86,88,88},{11,12,14},{17,17,21},{13,12,17},{11,10,15},{10,9,14},{13,13,17},{22,22,26},{22,21,26},{21,21,26},{44,48,54},{46,51,56},{30,33,38},{70,73,77},{214,214,214},{253,253,250},{250,251,247},{251,252,247},{251,252,247},{251,252,247},{252,253,248},{252,253,248},{251,251,249}},{{224,224,226},{224,224,226},{226,226,228},{228,229,229},{229,231,230},{229,231,230},{229,231,230},{233,235,233},{208,210,208},{62,64,65},{10,10,14},{11,10,15},{11,10,15},{11,10,15},{9,8,13},{9,8,13},{9,7,12},{10,8,13},{14,13,18},{29,31,36},{27,30,36},{28,32,39},{45,49,56},{168,169,172},{252,252,252},{250,250,247},{250,251,246},{251,252,246},{251,252,247},{251,252,247},{251,252,247},{251,251,249}},{{224,224,226},{224,224,226},{225,225,226},{225,226,226},{226,228,227},{225,227,226},{227,229,228},{230,232,230},{200,201,202},{57,58,62},{10,11,16},{11,9,14},{11,9,14},{14,12,17},{13,10,15},{9,7,12},{8,6,11},{10,8,13},{9,7,12},{22,21,26},{27,29,35},{38,42,51},{46,52,62},{115,115,122},{236,235,237},{251,251,248},{250,251,245},{251,252,246},{251,252,247},{251,252,247},{251,252,247},{251,251,249}},{{223,224,225},{223,224,225},{227,227,229},{230,231,232},{230,231,232},{222,224,224},{191,193,193},{156,157,158},{139,140,142},{44,45,48},{9,10,15},{12,9,15},{13,10,15},{13,10,15},{15,11,17},{16,12,18},{11,7,13},{11,7,13},{9,6,12},{15,14,19},{25,27,33},{41,46,55},{52,58,69},{49,52,61},{165,166,170},{252,254,252},{250,251,246},{251,252,246},{251,252,247},{251,252,247},{251,252,247},{251,251,249}},{{220,222,222},{220,222,221},{182,183,187},{158,159,163},{158,159,162},{121,122,126},{77,78,81},{40,41,45},{32,33,37},{14,15,19},{8,8,12},{10,8,12},{11,9,12},{11,9,12},{12,9,13},{14,12,17},{13,10,16},{9,6,12},{7,4,10},{8,7,12},{18,19,24},{32,35,42},{40,45,54},{32,40,46},{66,73,76},{171,175,176},{244,246,244},{251,252,248},{252,253,247},{252,253,247},{252,253,248},{252,252,250}},{{219,221,220},{206,208,206},{96,97,101},{39,39,46},{47,48,55},{35,35,42},{34,34,41},{30,30,35},{22,23,27},{19,20,24},{7,8,12},{8,6,9},{10,8,11},{9,7,10},{9,7,10},{8,6,10},{8,6,11},{6,4,9},{7,5,10},{9,8,13},{10,10,15},{19,21,27},{35,39,47},{48,56,62},{39,47,50},{59,64,66},{197,200,201},{253,254,251},{251,252,247},{252,253,248},{252,253,249},{253,253,252}},{{223,225,225},{186,188,186},{60,61,65},{25,25,33},{30,30,39},{35,35,44},{34,34,42},{31,32,37},{27,28,31},{25,26,30},{16,17,21},{7,6,9},{11,9,12},{11,9,12},{8,6,9},{7,5,9},{7,5,9},{6,4,8},{8,6,9},{12,10,13},{10,9,13},{10,10,16},{35,38,44},{47,54,59},{53,60,64},{46,51,55},{113,118,122},{241,242,241},{252,252,249},{251,251,249},{252,252,249},{252,252,251}},{{227,228,228},{198,200,198},{80,81,84},{39,39,45},{30,31,37},{23,24,30},{20,20,26},{23,24,28},{23,24,28},{24,25,29},{19,20,24},{10,8,12},{9,6,9},{11,9,12},{5,3,6},{6,4,6},{6,4,6},{6,4,6},{8,6,8},{13,9,11},{10,7,10},{11,10,15},{23,24,30},{36,39,44},{49,54,59},{52,59,65},{69,76,82},{212,213,214},{254,254,253},{250,250,249},{250,250,249},{251,251,250}},{{225,227,226},{223,225,223},{180,181,184},{150,152,154},{119,121,121},{78,80,80},{48,49,50},{26,27,31},{17,17,22},{13,13,18},{12,13,17},{14,13,16},{8,6,9},{6,4,7},{4,2,5},{5,3,4},{5,3,3},{4,2,3},{8,6,7},{14,10,10},{11,7,9},{12,10,14},{18,18,23},{24,25,30},{45,49,54},{60,68,76},{50,58,67},{163,165,167},{250,250,249},{249,249,249},{250,250,250},{250,250,250}},{{222,223,224},{222,224,223},{225,226,229},{226,228,228},{218,220,219},{199,201,200},{169,172,171},{133,135,137},{96,98,101},{62,64,67},{39,41,44},{29,29,31},{18,16,19},{8,6,9},{4,3,5},{4,2,4},{6,4,6},{7,5,6},{10,8,9},{13,9,10},{14,10,12},{16,14,17},{15,15,19},{20,20,25},{28,31,37},{44,49,57},{47,54,63},{96,100,105},{224,225,227},{251,251,252},{250,249,249},{250,249,250}},{{216,217,219},{211,212,214},{211,212,214},{212,214,215},{211,214,215},{208,210,211},{199,201,202},{185,187,189},{162,165,167},{128,130,132},{87,89,91},{63,63,65},{53,53,55},{43,43,45},{33,32,34},{24,24,26},{21,20,22},{18,17,19},{14,14,15},{9,8,10},{11,10,12},{13,13,15},{12,12,15},{25,25,31},{23,23,31},{20,21,28},{34,36,44},{65,71,81},{177,181,189},{251,251,254},{251,247,249},{250,249,250}},{{202,203,205},{183,184,186},{165,166,168},{147,148,150},{135,136,138},{138,139,141},{141,142,144},{140,141,143},{133,134,136},{118,119,121},{93,94,96},{73,73,75},{70,70,72},{69,69,71},{67,67,69},{62,62,64},{57,57,59},{63,63,65},{50,51,53},{42,42,44},{37,37,39},{24,24,26},{15,15,17},{17,18,23},{17,18,24},{12,13,19},{19,20,26},{37,42,52},{107,111,119},{229,228,232},{251,248,250},{248,247,249}},{{194,195,197},{168,169,171},{134,135,137},{94,94,96},{70,70,72},{68,68,70},{70,70,72},{71,71,73},{72,72,74},{72,72,74},{67,67,69},{59,59,61},{58,58,60},{60,60,62},{61,61,63},{62,62,64},{78,78,80},{92,92,94},{99,99,101},{102,102,104},{115,115,117},{121,121,123},{116,116,118},{104,104,109},{90,91,95},{75,76,81},{54,55,60},{62,64,71},{112,112,119},{212,211,215},{251,249,251},{247,246,248}},{{187,188,190},{160,161,163},{124,125,127},{84,83,86},{66,65,67},{62,61,64},{59,58,61},{58,57,59},{56,55,58},{54,54,56},{53,52,55},{52,52,54},{54,54,56},{56,56,58},{57,57,59},{60,60,62},{71,71,73},{77,77,79},{96,96,98},{105,105,107},{123,123,125},{150,150,152},{175,175,177},{192,193,196},{202,203,206},{202,203,206},{189,190,193},{194,194,198},{217,216,220},{239,237,241},{248,246,249},{247,247,249}},{{190,191,193},{160,162,163},{120,121,123},{81,79,82},{68,65,69},{65,62,65},{61,59,62},{58,56,59},{55,52,56},{52,50,53},{51,49,52},{49,49,51},{50,50,52},{51,52,54},{54,54,56},{54,54,56},{52,52,54},{58,58,60},{75,75,77},{92,92,94},{112,112,114},{136,136,138},{160,160,162},{183,183,185},{203,204,206},{224,225,227},{243,243,245},{251,248,250},{249,246,248},{246,244,247},{246,244,248},{246,245,248}},{{206,207,210},{184,185,188},{149,149,152},{104,102,105},{73,71,73},{64,62,64},{61,60,62},{60,57,60},{57,53,57},{55,51,55},{53,49,53},{51,49,52},{50,50,52},{50,49,52},{50,50,52},{48,48,50},{48,47,50},{53,52,54},{64,64,66},{84,83,85},{105,105,107},{129,129,131},{152,152,154},{174,175,177},{194,195,197},{213,214,216},{228,228,230},{239,237,239},{244,241,244},{244,243,246},{245,244,248},{245,244,247}},{{218,217,222},{210,209,214},{194,194,198},{161,162,162},{115,117,116},{79,80,79},{60,62,61},{57,58,59},{57,56,58},{57,52,56},{55,50,54},{52,49,53},{50,48,51},{49,48,51},{49,48,51},{50,49,52},{51,49,52},{54,52,55},{61,59,62},{78,77,79},{98,98,100},{120,120,122},{143,143,145},{166,166,168},{187,187,189},{208,208,210},{224,224,226},{236,235,239},{241,240,245},{243,242,247},{244,242,247},{243,243,247}},{{219,218,223},{217,216,221},{216,216,220},{206,207,208},{182,184,184},{144,146,145},{100,102,102},{70,71,72},{57,56,58},{54,51,54},{54,49,53},{51,49,52},{49,48,51},{49,47,50},{50,48,51},{52,50,53},{54,52,55},{55,53,56},{59,57,60},{72,72,74},{92,92,94},{111,111,113},{133,133,135},{156,156,158},{179,179,181},{200,200,202},{217,217,219},{230,230,234},{238,237,242},{241,240,245},{241,240,245},{241,240,245}},{{217,216,221},{216,215,220},{217,216,221},{216,217,219},{212,213,214},{198,199,201},{170,171,172},{131,133,135},{94,95,97},{69,68,70},{58,55,58},{53,51,54},{52,50,53},{52,50,53},{51,49,52},{52,50,53},{54,52,55},{55,53,56},{59,57,60},{69,68,70},{86,86,88},{104,104,106},{125,125,127},{147,147,149},{172,172,174},{193,193,195},{210,210,212},{224,223,227},{233,232,237},{239,238,243},{239,238,243},{240,238,243}}} -1|1|'dog'|'0/img2.jpg'|{{{126,118,110},{122,115,108},{126,119,111},{127,119,109},{130,122,111},{130,122,111},{132,124,113},{133,125,114},{130,122,111},{132,124,113},{134,126,115},{131,123,112},{131,123,112},{134,126,115},{133,125,114},{136,128,117},{137,129,118},{137,129,118},{136,128,117},{131,123,112},{130,122,111},{132,124,113},{132,124,113},{132,124,113},{129,122,110},{127,121,109},{127,121,109},{125,119,107},{124,118,106},{124,118,106},{120,114,102},{117,111,99}},{{122,115,107},{119,112,104},{121,114,106},{124,116,107},{129,121,110},{130,122,111},{130,122,111},{130,122,111},{128,120,109},{131,123,112},{134,126,115},{132,124,113},{132,124,113},{134,126,115},{136,128,117},{139,131,120},{141,133,122},{138,130,119},{135,127,116},{130,122,111},{131,123,112},{132,124,113},{133,125,114},{133,125,114},{131,124,113},{129,123,111},{130,124,112},{129,123,111},{127,121,109},{124,118,106},{122,116,104},{117,111,99}},{{122,115,107},{121,114,106},{121,114,106},{125,118,108},{128,120,109},{129,121,110},{131,123,112},{130,122,111},{128,120,109},{131,123,112},{133,125,114},{134,126,115},{136,128,117},{137,129,118},{138,130,119},{139,131,120},{139,131,120},{139,131,120},{139,131,120},{139,131,120},{139,131,120},{137,129,118},{137,129,118},{138,130,119},{136,129,118},{135,129,117},{133,127,115},{130,124,112},{129,123,111},{126,120,108},{123,117,105},{121,115,103}},{{123,116,108},{123,116,108},{125,118,110},{127,119,109},{127,119,108},{129,121,110},{132,125,113},{131,123,112},{130,122,111},{134,127,115},{134,126,115},{135,127,116},{133,125,114},{134,126,116},{136,128,118},{140,132,121},{142,134,123},{141,133,122},{141,132,122},{142,133,123},{141,132,122},{141,133,122},{140,131,120},{137,128,117},{136,128,117},{136,130,118},{132,127,115},{128,122,110},{127,121,109},{127,121,109},{125,119,107},{123,117,105}},{{124,119,110},{125,119,111},{128,122,113},{129,123,113},{132,123,113},{136,123,115},{137,125,116},{134,125,116},{135,125,116},{136,125,116},{134,125,115},{133,126,116},{136,124,115},{135,127,113},{137,131,115},{140,130,118},{141,130,120},{139,131,120},{137,132,118},{135,133,117},{134,130,117},{136,130,118},{136,130,119},{134,129,117},{132,127,115},{132,126,114},{131,125,113},{131,124,112},{129,122,110},{127,121,109},{128,122,110},{128,122,110}},{{124,121,112},{124,121,112},{128,124,115},{129,125,116},{136,124,118},{145,124,121},{143,126,121},{136,128,120},{136,126,119},{140,123,118},{136,126,118},{133,129,121},{146,130,130},{136,130,115},{135,133,112},{138,131,117},{138,130,119},{139,132,120},{137,132,117},{133,132,113},{132,131,115},{135,130,118},{136,129,118},{135,130,118},{131,127,115},{132,124,113},{132,124,113},{131,123,112},{129,122,110},{128,122,110},{127,121,109},{127,121,109}},{{123,120,111},{122,119,110},{124,121,112},{126,122,113},{132,122,115},{138,125,119},{135,128,120},{128,130,119},{130,129,119},{134,124,117},{127,118,110},{129,126,120},{142,135,139},{154,139,136},{141,130,123},{130,133,125},{131,135,121},{138,135,122},{143,130,122},{145,124,120},{142,126,119},{140,128,118},{141,128,119},{142,126,118},{139,125,116},{134,126,115},{134,126,115},{133,125,114},{131,123,112},{129,122,110},{126,120,108},{124,118,106}},{{121,118,109},{121,118,109},{122,119,110},{123,118,109},{127,121,113},{130,126,117},{131,131,121},{134,136,125},{130,131,121},{133,130,121},{133,127,119},{136,131,123},{115,104,98},{128,91,93},{151,118,117},{139,136,125},{135,137,122},{140,136,124},{139,132,124},{141,131,126},{140,128,121},{136,129,117},{134,131,118},{135,128,117},{137,126,116},{135,127,116},{135,127,116},{133,125,114},{131,123,112},{128,122,110},{127,121,109},{125,119,107}},{{124,121,112},{124,121,112},{126,123,114},{127,122,113},{128,124,115},{132,130,121},{138,131,123},{162,149,143},{156,144,138},{137,133,124},{141,136,128},{143,136,125},{137,120,105},{119,80,78},{144,108,108},{147,136,125},{142,133,123},{146,133,124},{149,134,123},{148,132,120},{151,130,121},{140,129,119},{127,134,118},{126,133,117},{131,128,116},{135,127,116},{134,126,115},{132,124,113},{130,122,111},{128,122,110},{128,122,110},{126,120,108}},{{126,123,114},{126,123,114},{128,125,116},{130,127,118},{129,129,119},{135,131,122},{143,122,119},{142,106,108},{163,131,132},{147,134,128},{140,136,127},{139,132,124},{141,134,128},{120,115,122},{134,131,142},{151,151,155},{143,139,133},{154,128,121},{152,90,85},{155,61,56},{158,77,80},{155,115,115},{141,131,121},{133,131,118},{135,128,117},{135,127,116},{134,126,115},{132,124,113},{130,123,112},{129,123,111},{126,120,108},{126,120,108}},{{126,121,112},{126,121,113},{130,125,116},{131,126,116},{131,128,117},{134,127,119},{141,123,120},{110,64,71},{121,57,65},{151,117,110},{143,135,122},{132,134,122},{136,138,130},{102,114,121},{80,97,121},{90,107,138},{130,146,164},{149,133,146},{142,71,84},{165,54,64},{151,34,43},{150,63,65},{159,116,110},{142,131,119},{134,130,118},{135,127,116},{136,128,117},{134,126,115},{131,123,112},{130,124,112},{126,120,108},{124,118,106}},{{129,122,114},{128,121,114},{131,124,116},{133,127,115},{135,129,117},{136,129,120},{138,132,127},{150,123,127},{144,96,100},{152,120,109},{144,129,114},{131,128,119},{171,174,177},{160,168,180},{132,146,167},{97,117,145},{106,131,159},{136,135,162},{150,112,134},{186,129,141},{202,129,128},{149,78,70},{159,118,107},{146,134,121},{135,132,119},{136,128,117},{136,128,117},{136,128,117},{134,126,115},{134,128,116},{131,125,113},{126,120,108}},{{133,126,118},{132,125,117},{134,127,119},{136,130,118},{136,130,118},{138,132,123},{139,131,126},{113,113,117},{126,135,146},{168,171,175},{149,142,143},{119,111,122},{148,157,188},{194,207,232},{166,178,191},{159,172,180},{180,194,207},{191,192,207},{170,156,167},{189,168,173},{211,196,191},{149,139,127},{147,135,123},{149,136,123},{145,134,122},{140,132,121},{138,130,119},{136,128,117},{132,125,113},{132,126,114},{131,125,113},{128,122,110}},{{134,127,119},{135,128,120},{136,128,121},{136,130,118},{137,131,119},{137,131,122},{142,134,128},{90,93,108},{33,52,92},{89,107,144},{145,161,193},{152,169,196},{116,138,164},{133,151,175},{167,177,200},{189,194,216},{207,211,225},{210,216,225},{204,212,217},{203,212,214},{173,170,171},{143,132,131},{146,137,129},{145,137,122},{144,136,120},{141,133,122},{139,131,120},{138,130,119},{133,125,114},{130,124,112},{128,122,110},{126,120,108}},{{131,124,116},{134,128,120},{135,128,120},{136,130,118},{139,133,121},{138,131,122},{109,101,95},{121,117,130},{103,109,150},{52,67,114},{60,87,130},{108,140,172},{150,174,198},{146,165,191},{128,141,172},{140,148,181},{158,164,187},{169,181,196},{183,201,208},{183,195,199},{154,140,144},{156,133,132},{150,137,128},{142,138,121},{141,138,121},{141,133,122},{137,129,118},{136,128,117},{135,127,116},{133,127,115},{129,123,111},{128,122,110}},{{132,124,116},{130,121,114},{135,126,120},{140,131,122},{140,134,122},{138,135,124},{125,119,115},{157,159,162},{162,174,189},{140,149,179},{140,154,186},{131,146,177},{99,114,155},{137,152,188},{164,182,209},{164,182,203},{156,170,195},{146,162,184},{147,161,179},{168,173,191},{159,159,158},{147,144,128},{146,139,124},{146,139,123},{143,135,123},{142,134,124},{136,128,117},{133,125,114},{134,127,116},{127,121,109},{129,123,111},{129,123,111}},{{141,129,119},{136,122,116},{135,124,120},{141,129,128},{135,130,122},{148,152,145},{185,191,204},{141,169,193},{119,147,170},{181,181,202},{192,200,212},{204,203,220},{167,172,199},{132,143,172},{119,136,168},{137,160,193},{169,188,213},{185,192,204},{175,177,202},{92,107,179},{97,120,166},{134,143,141},{149,140,126},{147,137,121},{140,132,126},{140,131,128},{138,131,116},{137,129,115},{137,128,119},{133,125,115},{131,124,112},{129,122,111}},{{139,129,117},{132,130,117},{127,134,120},{131,137,124},{157,154,155},{180,178,199},{118,136,175},{89,115,149},{153,146,163},{192,179,185},{182,207,197},{193,217,214},{202,206,219},{193,200,218},{171,186,209},{129,149,182},{96,125,171},{137,161,179},{169,183,203},{53,62,153},{34,44,157},{102,112,172},{124,127,141},{145,138,133},{143,133,125},{140,133,117},{139,132,117},{137,129,117},{136,128,116},{137,129,118},{135,127,116},{130,122,111}},{{134,128,114},{137,129,112},{143,130,112},{145,90,76},{165,123,127},{107,118,147},{94,115,158},{162,137,167},{152,76,85},{165,113,119},{207,202,202},{200,212,217},{213,213,226},{223,226,236},{220,229,236},{210,224,231},{150,175,200},{71,107,153},{104,138,195},{107,119,184},{50,51,159},{70,83,198},{60,77,162},{114,111,149},{140,126,118},{136,130,103},{133,125,112},{136,127,117},{134,126,110},{135,127,115},{134,126,115},{129,121,110}},{{137,127,114},{154,108,97},{159,66,64},{168,25,19},{151,29,30},{117,61,76},{168,148,162},{201,168,175},{180,136,140},{192,168,177},{222,207,225},{228,224,240},{219,216,222},{187,186,189},{189,190,189},{214,215,210},{233,229,229},{181,198,221},{77,114,170},{93,124,184},{133,149,200},{68,82,181},{33,46,185},{86,86,176},{134,125,146},{137,128,123},{136,128,117},{135,128,113},{134,126,112},{132,124,113},{131,123,112},{129,121,110}},{{139,122,111},{133,65,55},{151,14,14},{167,15,13},{174,17,17},{171,20,23},{155,53,47},{191,142,135},{216,219,216},{213,237,237},{214,218,224},{166,172,170},{95,92,90},{46,40,39},{51,45,44},{82,74,75},{125,112,115},{164,166,160},{147,167,173},{55,80,130},{118,138,172},{139,150,196},{64,67,163},{104,112,203},{113,111,187},{121,103,148},{140,131,120},{136,129,106},{133,124,120},{130,122,113},{129,121,110},{127,119,108}},{{140,121,112},{133,85,73},{137,45,37},{147,29,31},{162,20,19},{180,18,10},{173,19,14},{161,56,61},{181,157,159},{148,152,149},{90,79,74},{44,39,25},{55,47,39},{69,59,54},{57,45,43},{43,31,31},{33,28,24},{34,29,26},{54,48,52},{58,57,67},{73,79,100},{142,144,158},{137,137,155},{113,126,182},{84,89,183},{102,86,158},{130,123,125},{135,128,110},{135,123,122},{131,121,111},{128,120,107},{126,118,106}},{{135,124,117},{106,91,83},{65,46,39},{114,58,55},{154,47,45},{162,31,28},{145,27,26},{99,25,26},{61,33,30},{39,31,22},{52,47,36},{97,89,77},{135,124,111},{143,132,119},{137,125,113},{126,114,102},{110,99,92},{93,81,78},{74,60,59},{63,51,47},{80,71,69},{114,110,107},{132,133,133},{129,139,151},{141,151,187},{166,169,211},{173,172,189},{140,133,131},{134,123,112},{133,123,107},{129,120,104},{127,119,104}},{{129,120,112},{68,60,52},{19,12,5},{33,27,22},{73,42,41},{81,26,28},{50,12,14},{32,15,11},{51,37,28},{92,85,73},{133,125,114},{151,137,127},{147,135,121},{144,132,118},{147,135,121},{149,137,123},{150,138,125},{148,136,124},{145,133,120},{137,125,113},{115,105,92},{85,79,69},{101,101,95},{128,133,130},{159,169,173},{199,211,219},{225,231,232},{184,185,176},{133,129,114},{129,122,107},{127,119,105},{125,117,104}},{{128,120,109},{109,101,91},{76,67,58},{25,33,23},{14,17,12},{30,18,16},{45,42,38},{78,81,71},{120,115,100},{140,131,118},{144,128,117},{145,126,116},{144,132,118},{144,132,118},{145,133,119},{145,133,119},{142,131,114},{145,134,115},{147,136,118},{148,137,119},{146,137,118},{128,122,105},{97,93,79},{90,89,78},{116,120,113},{155,163,161},{200,206,205},{220,225,226},{178,181,181},{127,125,116},{122,118,107},{124,117,104}},{{125,117,104},{124,116,104},{129,122,109},{99,95,79},{72,63,51},{93,76,70},{126,111,100},{136,130,113},{136,133,114},{142,128,113},{135,116,103},{129,112,99},{130,118,104},{137,125,111},{142,130,116},{144,132,118},{142,131,113},{144,133,114},{145,134,115},{145,134,115},{144,134,117},{144,135,119},{142,134,118},{112,104,89},{79,72,64},{96,92,94},{147,146,154},{194,197,211},{222,228,241},{170,174,170},{119,119,110},{123,117,104}},{{124,117,101},{120,113,97},{125,118,103},{131,121,104},{130,120,109},{136,124,119},{143,126,117},{139,127,110},{139,132,113},{146,129,114},{142,125,109},{132,123,105},{134,123,109},{141,129,115},{142,130,116},{143,131,117},{141,129,114},{144,132,117},{145,133,118},{144,132,117},{140,128,116},{140,128,117},{144,131,119},{145,131,118},{131,115,104},{99,84,77},{86,77,74},{112,112,114},{166,172,176},{203,211,209},{153,156,149},{120,115,103}},{{124,117,103},{122,115,101},{130,122,109},{130,122,109},{127,122,112},{130,125,118},{134,125,116},{138,128,114},{140,130,113},{142,127,112},{140,126,111},{138,129,113},{142,130,116},{139,127,113},{134,122,108},{142,130,116},{141,129,116},{142,130,117},{144,132,119},{143,131,118},{133,122,111},{136,126,116},{140,128,118},{139,127,116},{141,127,113},{140,126,110},{119,110,96},{92,88,75},{81,81,71},{120,120,114},{156,154,145},{123,117,105}},{{126,117,106},{128,120,109},{130,122,111},{129,121,110},{131,122,111},{132,124,113},{133,125,114},{136,126,114},{139,127,113},{138,126,112},{138,126,112},{140,128,114},{139,127,113},{137,125,111},{135,123,109},{142,130,116},{140,128,114},{140,128,114},{140,128,114},{140,128,114},{136,126,114},{136,128,117},{137,129,118},{137,129,118},{134,126,114},{134,127,111},{137,130,115},{136,128,113},{119,112,97},{97,89,78},{113,105,93},{121,113,102}},{{126,118,107},{126,118,107},{128,120,109},{130,122,111},{130,122,111},{131,123,112},{134,126,115},{136,126,114},{137,125,111},{137,125,111},{136,124,110},{139,127,113},{139,127,113},{139,127,113},{137,125,111},{138,126,112},{138,126,112},{138,126,112},{139,127,113},{139,127,113},{135,125,113},{138,130,119},{142,134,123},{141,133,122},{140,132,120},{138,130,117},{138,130,117},{135,127,114},{138,130,118},{133,125,114},{122,114,103},{114,106,95}},{{125,117,106},{126,118,107},{127,119,108},{129,121,110},{129,121,110},{129,121,110},{133,125,114},{133,123,111},{134,122,108},{135,123,109},{136,124,110},{138,126,112},{138,126,112},{136,124,110},{135,123,109},{136,124,110},{136,124,110},{137,125,111},{139,127,113},{138,126,112},{136,127,115},{138,130,119},{142,134,123},{140,132,121},{138,130,119},{134,126,115},{135,127,116},{135,127,116},{132,124,113},{128,120,109},{121,113,102},{114,106,95}},{{123,115,104},{123,115,104},{126,118,107},{128,120,109},{129,121,110},{129,121,110},{131,123,112},{132,122,110},{133,121,107},{135,123,109},{135,123,109},{135,123,109},{136,124,110},{133,121,107},{134,122,108},{136,124,110},{136,124,110},{137,125,111},{134,122,108},{130,118,104},{133,123,111},{133,125,115},{136,128,117},{136,128,117},{135,127,117},{134,125,116},{132,123,114},{131,122,113},{129,121,111},{128,119,109},{126,118,107},{121,113,102}}} -\. - -DROP TABLE IF EXISTS cifar_10_sample_batched; -DROP TABLE IF EXISTS cifar_10_sample_batched_summary; -SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_batched','y','x', 1, 255); - -DROP TABLE IF EXISTS cifar_10_sample_val, cifar_10_sample_val_summary; -SELECT validation_preprocessor_dl('cifar_10_sample','cifar_10_sample_val','y','x', 'cifar_10_sample_batched', 1); ---- NOTE: In order to test fit_merge, we need at least 2 rows in the batched table (1 on each segment). - -DROP TABLE IF EXISTS cifar_10_sample_int_batched; -DROP TABLE IF EXISTS cifar_10_sample_int_batched_summary; -SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_int_batched','y','x', 2, 255, 5); - --- This table is for testing a different input shape (3, 32, 32) instead of (32, 32, 3). --- Create a table with image shape 3, 32, 32 -drop table if exists cifar_10_sample_test_shape; -create table cifar_10_sample_test_shape(id INTEGER, y SMALLINT, x REAL[] ); -copy cifar_10_sample_test_shape from stdin delimiter '|'; -1|0|{{{248,248,250,245,245,246,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245},{247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245},{245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247},{248,248,250,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245},{247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245},{245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247,245,245,247},{249,249,251,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246},{248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248},{249,249,251,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246},{248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248},{249,249,251,245,245,247,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,249,246,246,249,246,246},{249,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,248,246,246,249,246,246,249,246,246,248,246,246,248,246,246,248,246,246,248,246,246,248},{249,249,251,245,245,247,246,246,248,247,246,249,247,246,248,246,246,247,245,246,248,246,246,251,247,245,252,247,245,251,247,245},{250,247,245,250,248,245,249,247,245,248,246,243,247,246,243,247,245,244,248,244,244,248,246,245,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,248,246,245,249,246,246,249,245,246,249,245,246,249,246,246,247,246,246,246,246,246,247},{249,249,251,245,245,247,246,246,248,247,246,248,247,246,248,247,246,249,246,246,250,245,246,250,245,246,248,245,246,247,246,246},{247,246,246,247,247,246,247,246,246,247,245,244,245,245,244,245,244,244,247,244,243,247,245,245,248,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,246,246,246,244,246,247,244,246,247,245,245,246,247,246,245,250,246,246,248,246,246,248},{249,249,251,245,245,247,246,246,248,244,245,245,242,243,246,244,243,248,246,244,247,247,247,246,248,250,247,247,249,247,246,247},{248,245,246,248,245,246,251,245,246,251,243,244,249,243,244,248,244,244,244,244,244,244,245,245,247,246,246,248,246,246,248,246},{246,248,246,246,248,246,246,248,246,246,247,246,247,245,246,246,244,247,245,243,249,247,246,246,245,249,246,245,251,246,245,250},{249,249,251,245,245,247,245,245,247,245,248,249,245,248,251,245,245,247,247,246,237,245,243,229,235,233,226,238,236,233,243,240},{241,248,245,249,246,245,253,246,245,253,245,244,252,245,245,251,245,246,243,245,246,241,246,246,246,246,245,249,246,246,249,246},{246,248,246,246,248,246,246,248,245,245,250,244,244,251,248,245,249,247,242,239,240,235,227,242,243,235,245,245,247,246,245,251},{249,249,251,244,244,246,233,233,235,187,192,193,161,166,169,160,161,157,208,207,179,230,227,188,215,210,179,215,209,184,231,224},{205,240,233,220,239,236,226,241,238,229,242,238,230,241,238,229,238,239,229,244,245,238,246,246,245,246,246,250,246,246,249,246},{246,248,247,246,248,247,246,248,245,245,250,246,247,252,240,238,232,222,216,193,202,198,164,206,209,180,241,242,235,246,245,250},{248,248,250,245,245,247,231,231,233,127,131,133,39,44,46,26,27,17,109,109,70,169,167,108,180,174,116,191,183,131,203,194},{147,184,175,133,145,139,102,149,144,106,150,146,108,151,148,114,186,187,169,240,241,234,245,246,245,246,245,250,246,246,251,245}},{{246,250,245,246,249,245,246,249,246,248,248,229,231,219,196,198,164,196,195,140,185,183,121,186,191,148,240,242,229,247,246,250},{249,249,251,244,244,246,244,244,246,233,232,234,174,173,172,114,113,100,112,112,85,112,110,70,112,108,59,125,119,67,151,143},{86,149,139,83,124,113,60,111,103,42,115,111,51,151,148,103,203,204,184,241,243,237,246,247,249,247,247,251,245,247,252,246},{246,251,248,247,249,240,243,241,216,219,204,196,197,157,195,200,138,197,199,132,178,178,118,208,210,176,246,245,244,245,245,250},{249,249,251,244,244,246,244,244,246,246,244,246,249,247,245,239,238,230,222,221,209,199,197,178,124,120,91,82,77,36,132,126},{71,171,164,102,171,162,100,143,137,68,147,143,80,194,191,145,201,204,180,211,214,204,227,229,226,236,238,235,245,243,239,249},{242,236,235,224,212,209,198,177,194,183,153,187,176,134,165,158,106,141,137,80,109,107,52,187,187,159,248,246,251,245,245,249},{248,248,250,244,244,246,244,244,246,245,244,244,242,242,241,243,243,242,250,249,249,253,251,251,189,185,176,91,87,61,96,92},{46,155,152,93,153,150,88,149,147,82,153,151,92,177,175,130,197,200,172,199,203,187,186,190,179,166,169,156,153,152,138,143},{133,117,115,96,73,123,102,69,145,124,86,132,108,74,90,68,36,80,68,26,100,98,47,213,212,190,249,247,251,244,245,248},{247,247,249,244,244,246,244,244,246,244,244,245,244,244,245,244,244,247,242,242,248,243,242,249,244,241,245,179,176,165,104,101},{73,82,80,40,89,90,43,101,102,53,108,107,60,120,118,78,130,134,105,144,149,131,161,165,150,154,159,140,118,124,106,87},{86,70,74,64,42,69,58,29,74,60,27,81,58,27,84,60,28,100,86,48,156,155,114,223,222,207,247,245,250,245,246,246},{247,247,249,244,243,246,244,244,246,243,244,247,242,245,247,242,245,247,242,245,247,243,245,247,244,244,248,241,240,240,219,216},{209,140,138,125,85,88,68,79,81,56,86,85,54,83,81,46,62,65,40,55,60,44,77,81,66,91,96,75,106,113,89,109},{112,90,104,100,78,88,87,63,72,72,45,70,62,26,94,79,29,142,132,87,149,149,121,205,203,195,246,244,250,244,245,244},{246,247,248,243,243,244,244,244,245,242,244,247,241,245,247,241,245,245,241,245,243,241,245,243,242,244,245,244,244,248,252,249},{254,212,210,213,185,186,184,160,161,153,159,159,140,107,107,79,61,64,42,49,53,38,41,45,31,38,42,25,38,41,18,58},{56,29,76,72,43,83,83,52,82,86,58,74,75,37,139,133,71,173,169,114,129,131,104,202,201,197,247,245,250,242,244,243},{245,247,245,242,243,242,242,244,243,242,244,245,242,243,245,242,243,245,242,244,244,242,243,244,242,243,245,242,243,246,242,242},{246,175,175,177,159,160,154,142,143,133,133,135,120,105,108,89,89,96,75,85,93,75,72,78,64,56,63,51,42,45,34,47},{47,26,55,55,18,67,65,21,79,74,38,74,68,30,117,113,58,176,174,122,138,137,108,177,178,176,247,247,250,243,244,245},{245,247,246,242,243,242,242,244,243,242,243,244,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,243,244},{245,211,213,212,210,215,207,178,181,171,133,134,122,108,107,94,89,97,80,76,88,69,77,89,69,80,91,72,63,69,51,68},{70,46,72,72,40,99,96,65,101,96,59,62,57,8,69,66,21,132,127,103,203,198,196,228,229,231,243,244,246,242,243,245},{245,246,248,241,243,244,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,242,243,245,241,242},{244,245,248,248,243,252,246,245,249,242,239,237,230,218,212,205,194,193,184,133,135,122,60,62,43,76,80,54,82,85,57,79},{81,55,117,117,97,201,199,188,171,168,147,95,92,54,130,128,99,212,209,204,245,241,252,245,245,248,242,243,245,241,242,244},{244,245,249,241,242,246,241,242,246,241,242,245,242,243,245,241,242,244,241,242,244,241,242,244,241,242,244,241,242,244,241,242},{244,239,242,243,235,244,242,240,244,241,240,238,235,192,184,181,182,170,169,152,140,136,53,42,30,68,57,38,93,90,66,139},{141,122,214,213,207,244,244,245,228,226,225,210,207,200,225,223,215,244,244,240,242,243,243,242,243,245,243,244,246,243,244,246},{243,244,249,241,241,246,241,242,247,241,242,245,241,242,243,241,242,244,241,242,244,241,242,244,241,242,244,241,242,244,241,242}},{{244,241,242,245,241,242,246,242,243,245,243,242,242,221,219,217,214,205,205,196,185,183,100,90,83,121,112,100,188,184,171,234},{232,228,245,245,249,238,242,244,243,245,248,248,245,253,244,243,247,240,243,241,240,244,239,242,243,244,242,243,245,242,243,245},{243,244,250,240,241,247,240,242,247,240,241,246,240,241,245,240,242,245,241,242,245,241,242,245,241,242,245,241,242,245,241,242},{245,241,242,245,242,241,247,242,241,246,242,243,245,246,247,248,248,248,248,246,245,244,209,209,207,225,225,221,247,246,244,244},{242,247,240,240,248,239,243,246,239,242,243,241,240,245,241,241,247,240,243,247,239,243,245,241,242,245,241,242,245,241,242,245},{241,244,251,238,241,247,238,241,248,238,241,248,239,241,248,239,241,248,239,242,248,239,241,247,240,241,246,240,241,246,241,242},{246,241,241,246,240,241,246,240,241,246,241,242,246,240,241,245,239,240,245,240,241,245,242,243,247,241,243,246,240,240,245,240},{240,246,241,241,247,240,241,246,240,242,246,241,242,246,241,242,247,241,242,247,241,242,247,241,242,247,241,242,247,241,242,247},{240,243,250,237,240,247,238,241,248,238,241,248,237,240,247,237,240,247,237,240,247,238,241,247,240,241,246,240,241,246,239,240},{245,239,240,245,240,241,246,240,241,246,239,240,245,239,241,246,240,241,246,240,241,247,239,240,245,240,241,246,241,242,247,240},{241,246,240,241,246,240,241,246,240,241,246,240,241,246,240,241,246,241,242,247,241,242,247,241,242,247,241,242,247,241,242,247},{239,242,249,237,240,247,238,240,247,237,240,247,236,239,246,236,239,246,236,239,246,237,240,246,238,241,246,237,241,245,237,240},{244,237,240,245,238,241,246,238,241,246,237,240,245,237,240,245,238,241,245,238,241,246,238,241,246,238,241,246,238,241,246,238},{241,246,238,241,246,238,241,246,238,241,246,238,241,245,238,241,246,239,242,247,239,242,247,239,242,247,239,242,247,239,242,247},{239,242,249,236,239,246,236,239,246,237,240,247,237,240,247,236,239,246,236,239,246,236,240,246,235,241,245,235,241,245,235,241},{245,235,241,245,235,241,245,235,241,245,235,241,245,235,241,245,235,241,245,235,241,245,235,241,245,234,241,244,235,241,245,235},{242,245,236,242,246,236,242,246,235,242,245,234,240,244,235,241,245,236,242,246,236,242,246,236,242,246,236,242,246,236,242,246},{239,242,249,236,239,245,236,239,246,236,239,246,236,239,246,236,239,246,236,239,246,235,240,245,233,241,244,233,241,244,233,241},{244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233,241,244,233},{241,244,234,242,245,235,243,246,234,242,245,233,241,244,234,242,245,235,243,246,235,243,246,235,243,246,235,243,246,235,243,246},{237,242,249,234,239,246,235,239,246,234,239,246,234,238,246,235,239,246,235,239,246,234,239,246,232,240,245,232,240,244,232,240},{244,232,240,244,232,240,244,232,240,244,232,240,244,232,240,244,232,240,244,232,240,245,233,240,245,233,241,245,233,240,245,232},{240,245,233,241,246,234,242,247,234,241,246,233,240,245,233,241,245,234,242,246,234,242,246,234,242,247,234,242,247,234,242,247},{235,242,250,232,238,246,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239},{247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,233,240,247,233,240,248,233,240,247,233},{240,247,233,240,248,233,240,248,233,240,247,232,239,247,232,239,247,232,239,247,232,239,247,233,240,248,233,240,248,233,240,248},{235,242,250,231,239,246,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239},{247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232},{239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,233,240,248},{235,242,250,232,238,246,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239},{247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,231,238,246,231,238,246,231},{238,246,231,238,246,231,238,246,232,239,247,232,239,247,232,239,247,232,239,247,232,239,247,231,238,246,232,239,246,232,239,247}}} -\. - -DROP TABLE IF EXISTS cifar_10_sample_test_shape_batched; -DROP TABLE IF EXISTS cifar_10_sample_test_shape_batched_summary; -SELECT training_preprocessor_dl('cifar_10_sample_test_shape','cifar_10_sample_test_shape_batched','y','x', NULL, 255, 3); - -DROP TABLE IF EXISTS model_arch; -SELECT load_keras_model('model_arch', - $${ - "class_name": "Sequential", - "keras_version": "2.1.6", - "config": [{ - "class_name": "Conv2D", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, - "name": "conv2d_1", - "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, - "dtype": "float32", "activation": "relu", "trainable": true, - "data_format": "channels_last", "filters": 32, "padding": "valid", - "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, - "bias_initializer": {"class_name": "Zeros", "config": {}}, - "batch_input_shape": [null, 32, 32, 3], "use_bias": true, - "activity_regularizer": null, "kernel_size": [3, 3]}}, - {"class_name": "MaxPooling2D", "config": {"name": "max_pooling2d_1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}}, - {"class_name": "Dropout", "config": {"rate": 0.25, "noise_shape": null, "trainable": true, "seed": null, "name": "dropout_1"}}, - {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten_1", "data_format": "channels_last"}}, - {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": - {"class_name": "Zeros", "config": {}}, "units": 2, "use_bias": true, "activity_regularizer": null} - }], "backend": "tensorflow"}$$); - -SELECT load_keras_model('model_arch', - $${ - "class_name": "Sequential", - "keras_version": "2.1.6", - "config": [{ - "class_name": "Conv2D", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, - "name": "conv2d_1", - "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, - "dtype": "float32", "activation": "relu", "trainable": true, - "data_format": "channels_last", "filters": 32, "padding": "valid", - "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, - "bias_initializer": {"class_name": "Zeros", "config": {}}, - "batch_input_shape": [null, 32, 32, 3], "use_bias": true, - "activity_regularizer": null, "kernel_size": [3, 3]}}, - {"class_name": "MaxPooling2D", "config": {"name": "max_pooling2d_1", "trainable": true, "data_format": "channels_last", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}}, - {"class_name": "Dropout", "config": {"rate": 0.25, "noise_shape": null, "trainable": true, "seed": null, "name": "dropout_1"}}, - {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten_1", "data_format": "channels_last"}}, - {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": - {"class_name": "Zeros", "config": {}}, "units": 5, "use_bias": true, "activity_regularizer": null} - }], "backend": "tensorflow"}$$); - -SELECT load_keras_model('model_arch', - $${ - "class_name": "Sequential", - "keras_version": "2.1.6", - "config": [{ - "class_name": "Conv2D", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, - "name": "conv2d_1", - "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, - "dtype": "float32", "activation": "relu", "trainable": true, - "data_format": "channels_first", "filters": 32, "padding": "valid", - "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, - "bias_initializer": {"class_name": "Zeros", "config": {}}, - "batch_input_shape": [null, 3, 32, 32], "use_bias": true, - "activity_regularizer": null, "kernel_size": [3, 3]}}, - {"class_name": "MaxPooling2D", "config": {"name": "max_pooling2d_1", "trainable": true, "data_format": "channels_first", "pool_size": [2, 2], "padding": "valid", "strides": [2, 2]}}, - {"class_name": "Dropout", "config": {"rate": 0.25, "noise_shape": null, "trainable": true, "seed": null, "name": "dropout_1"}}, - {"class_name": "Flatten", "config": {"trainable": true, "name": "flatten_1", "data_format": "channels_first"}}, - {"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", "trainable": true, "kernel_regularizer": null, "bias_initializer": - {"class_name": "Zeros", "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null} - }], "backend": "tensorflow"}$$); - diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_evaluate.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_evaluate.sql_in deleted file mode 100644 index cdda44a..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_evaluate.sql_in +++ /dev/null @@ -1,117 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -m4_include(`SQLCommon.m4') - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_cifar.setup.sql_in' -) - --- -- Please do not break up the compile_params string --- -- It might break the assertion -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_saved_out', - 'model_arch', - 1, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['mae']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3); - --- Test that evaluate works as expected: -DROP TABLE IF EXISTS evaluate_out; -SELECT madlib_keras_evaluate('keras_saved_out', 'cifar_10_sample_val', 'evaluate_out', FALSE); - -SELECT assert(loss >= 0 AND - metric >= 0 AND - metrics_type = '{mae}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out)) -FROM evaluate_out; - --- Test that passing NULL / None instead of 0 for gpus_per_host works -DROP TABLE IF EXISTS evaluate_out; -SELECT madlib_keras_evaluate('keras_saved_out', 'cifar_10_sample_val', 'evaluate_out'); -SELECT assert(loss >= 0 AND - metric >= 0 AND - metrics_type = '{mae}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out)) -FROM evaluate_out; - --- Test that evaluate errors out correctly if mst_key is given for non-multi model tables -DROP TABLE IF EXISTS evaluate_out; -SELECT assert(trap_error($TRAP$ - SELECT madlib_keras_evaluate('keras_saved_out', 'cifar_10_sample_val', 'evaluate_out', FALSE ,1); - $TRAP$) = 1, 'Should error out if mst_key is given for non-multi model tables'); - --- Test that evaluate errors out correctly if model_arch field missing from fit output -DROP TABLE IF EXISTS evaluate_out; -ALTER TABLE keras_saved_out DROP COLUMN model_arch; -SELECT assert(trap_error($TRAP$ - SELECT madlib_keras_evaluate('keras_saved_out', 'cifar_10_sample_val', 'evaluate_out'); - $TRAP$) = 1, 'Should error out if model_arch column is missing from model_table'); - - -m4_changequote(`') -m4_ifdef(, , ) diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in deleted file mode 100644 index a35eb6b..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_fit.sql_in +++ /dev/null @@ -1,419 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_cifar.setup.sql_in' -) - -m4_include(`SQLCommon.m4') - --- Please do not break up the compile_params string --- It might break the assertion -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_saved_out', - 'model_arch', - 1, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['mae']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3, - NULL, - 'cifar_10_sample_val'); - -SELECT assert( - model_arch_table = 'model_arch' AND - model_id = 1 AND - model_type = 'madlib_keras' AND - start_training_time < now() AND - end_training_time > start_training_time AND - source_table = 'cifar_10_sample_batched' AND - validation_table = 'cifar_10_sample_val' AND - model = 'keras_saved_out' AND - dependent_varname = 'y' AND - dependent_vartype = 'smallint' AND - independent_varname = 'x' AND - normalizing_const = 255.0 AND - pg_typeof(normalizing_const) = 'real'::regtype AND - name is NULL AND - description is NULL AND - model_size > 0 AND - madlib_version is NOT NULL AND - compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['mae']$$::text AND - fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND - num_iterations = 3 AND - metrics_compute_frequency = 3 AND - num_classes = 2 AND - class_values = '{0,1}' AND - metrics_type = '{mae}' AND - training_metrics_final >= 0 AND - training_loss_final >= 0 AND - array_upper(training_metrics, 1) = 1 AND - array_upper(training_loss, 1) = 1 AND - array_upper(metrics_elapsed_time, 1) = 1 AND - validation_metrics_final >= 0 AND - validation_loss_final >= 0 AND - array_upper(validation_metrics, 1) = 1 AND - array_upper(validation_loss, 1) = 1 , - 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM keras_saved_out_summary) summary; - -SELECT assert( - model_weights IS NOT NULL AND - model_arch IS NOT NULL, 'Keras model output validation failed. Actual:' || __to_char(k)) -FROM (SELECT * FROM keras_saved_out) k; - --- Verify number of iterations for which metrics and loss are computed -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_saved_out', - 'model_arch', - 1, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3, - NULL, - 'cifar_10_sample_val', - 2); -SELECT assert( - num_iterations = 3 AND - metrics_compute_frequency = 2 AND - training_metrics_final >= 0 AND - training_loss_final >= 0 AND - metrics_type = '{accuracy}' AND - array_upper(training_metrics, 1) = 2 AND - array_upper(training_loss, 1) = 2 AND - array_upper(metrics_elapsed_time, 1) = 2 AND - validation_metrics_final >= 0 AND - validation_loss_final >= 0 AND - array_upper(validation_metrics, 1) = 2 AND - array_upper(validation_loss, 1) = 2 , - 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM keras_saved_out_summary) summary; - --- Test for - -- Non null name and description columns - -- Null validation table -DROP TABLE IF EXISTS keras_out, keras_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_out', - 'model_arch', - 1, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 2, - NULL, - NULL, - 1, - NULL, - 'model name', - 'model desc'); - -SELECT assert( - source_table = 'cifar_10_sample_batched' AND - model = 'keras_out' AND - dependent_varname = 'y' AND - independent_varname = 'x' AND - model_arch_table = 'model_arch' AND - model_id = 1 AND - compile_params = $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text AND - fit_params = $$ batch_size=2, epochs=1, verbose=0 $$::text AND - num_iterations = 2 AND - validation_table is NULL AND - metrics_compute_frequency = 1 AND - name = 'model name' AND - description = 'model desc' AND - model_type = 'madlib_keras' AND - model_size > 0 AND - start_training_time < now() AND - end_training_time > start_training_time AND - array_upper(metrics_elapsed_time, 1) = 2 AND - dependent_vartype = 'smallint' AND - madlib_version is NOT NULL AND - num_classes = 2 AND - class_values = '{0,1}' AND - metrics_type = '{accuracy}' AND - normalizing_const = 255.0 AND - training_metrics_final is not NULL AND - training_loss_final is not NULL AND - array_upper(training_metrics, 1) = 2 AND - array_upper(training_loss, 1) = 2 AND - validation_metrics_final is NULL AND - validation_loss_final is NULL AND - validation_metrics is NULL AND - validation_loss is NULL, - 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM keras_out_summary) summary; - -SELECT assert(model_weights IS NOT NULL , 'Keras model output validation failed') FROM (SELECT * FROM keras_out) k; - --- Validate metrics=NULL works with fit -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( -'cifar_10_sample_batched', -'keras_saved_out', -'model_arch', -1, -$$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy'$$::text, -$$ batch_size=2, epochs=1, verbose=0 $$::text, -1); - -SELECT assert( - metrics_type is NULL AND - training_metrics IS NULL AND - array_upper(training_loss, 1) = 1 AND - array_upper(metrics_elapsed_time, 1) = 1 AND - validation_metrics_final IS NULL AND - validation_loss_final >= 0 AND - validation_metrics IS NULL AND - array_upper(validation_loss, 1) = 1, - 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM keras_saved_out_summary) summary; - --- Validate metrics=[] works with fit -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( -'cifar_10_sample_batched', -'keras_saved_out', -'model_arch', -1, -$$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=[]$$::text, -$$ batch_size=2, epochs=1, verbose=0 $$::text, -1); - -SELECT assert( - metrics_type IS NULL AND - training_metrics IS NULL AND - array_upper(training_loss, 1) = 1 AND - array_upper(metrics_elapsed_time, 1) = 1 AND - validation_metrics_final IS NULL AND - validation_loss_final >= 0 AND - validation_metrics IS NULL AND - array_upper(validation_loss, 1) = 1, - 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM keras_saved_out_summary) summary; - --- Compile and fit parameter tests -DROP TABLE IF EXISTS keras_out, keras_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_out', - 'model_arch', - 1, - $$ optimizer='SGD', loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 1, - NULL, - NULL, - NULL, - NULL, 'model name', 'model desc'); - -DROP TABLE IF EXISTS keras_out, keras_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_out', - 'model_arch', - 1, - $$ optimizer='Adam()', loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 1, - NULL, - NULL, - NULL, - NULL, 'model name', 'model desc'); - -DROP TABLE IF EXISTS keras_out, keras_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_out', - 'model_arch', - 1, - $$ optimizer=Adam(epsilon=None), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 1, - FALSE, - NULL, - NULL, - NULL, 'model name', 'model desc'); - -DROP TABLE IF EXISTS keras_out, keras_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_out', - 'model_arch', - 1, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), metrics=['accuracy'], loss_weights=[2], sample_weight_mode=None, loss='categorical_crossentropy' $$::text, - $$ epochs=10, verbose=0, shuffle=True, initial_epoch=1, steps_per_epoch=2 $$::text, - 1, - NULL, - NULL, - NULL, - False, 'model name', 'model desc'); - -DROP TABLE IF EXISTS keras_out, keras_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_out', - 'model_arch', - 1, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), metrics=['accuracy'], loss_weights=[2], sample_weight_mode=None, loss='categorical_crossentropy' $$::text, - NULL, - 1, - NULL, - NULL, - NULL, - False, 'model name', 'model desc'); - --- -- negative test case for passing non numeric y to fit --- induce failure by passing a non numeric column -DROP TABLE IF EXISTS cifar_10_sample_val_failure; -CREATE TABLE cifar_10_sample_val_failure AS SELECT * FROM cifar_10_sample_val; -ALTER TABLE cifar_10_sample_val_failure rename dependent_var to dependent_var_original; -ALTER TABLE cifar_10_sample_val_failure rename buffer_id to dependent_var; -DROP TABLE IF EXISTS keras_out, keras_out_summary; -SELECT assert(trap_error($TRAP$SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_out', - 'model_arch', - 1, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 2, - NULL, - 'cifar_10_sample_val_failure');$TRAP$) = 1, - 'Passing y of type non numeric array to fit should error out.'); - --- Tests with text class values: --- Modify input data to have text classes, and mini-batch it. --- Create a new table using the text based column for dep var. -DROP TABLE IF EXISTS cifar_10_sample_text_batched; -m4_changequote(`') -CREATE TABLE cifar_10_sample_text_batched AS - SELECT buffer_id, independent_var, dependent_var, - independent_var_shape, dependent_var_shape - m4_ifdef(, , ) - FROM cifar_10_sample_batched m4_ifdef(, , ); - --- Insert a new row with NULL as the dependent var (one-hot encoded) -UPDATE cifar_10_sample_text_batched - SET dependent_var = convert_array_to_bytea(ARRAY[0,0,1,0,0]::smallint[]) WHERE buffer_id=0; -UPDATE cifar_10_sample_text_batched - SET dependent_var = convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) WHERE buffer_id=1; -INSERT INTO cifar_10_sample_text_batched(m4_ifdef(, , ) buffer_id, independent_var, dependent_var, independent_var_shape, dependent_var_shape) - SELECT m4_ifdef(, , ) 2 AS buffer_id, independent_var, - convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) AS dependent_var, - independent_var_shape, dependent_var_shape - FROM cifar_10_sample_batched WHERE cifar_10_sample_batched.buffer_id=0; -UPDATE cifar_10_sample_text_batched SET dependent_var_shape = ARRAY[1,5]; - --- Create the necessary summary table for the batched input. -DROP TABLE IF EXISTS cifar_10_sample_text_batched_summary; -CREATE TABLE cifar_10_sample_text_batched_summary( - source_table text, - output_table text, - dependent_varname text, - independent_varname text, - dependent_vartype text, - class_values text[], - buffer_size integer, - normalizing_const numeric); -INSERT INTO cifar_10_sample_text_batched_summary values ( - 'cifar_10_sample', - 'cifar_10_sample_text_batched', - 'y_text', - 'x', - 'text', - ARRAY[NULL,'cat','dog',NULL,NULL], - 1, - 255.0); - -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_text_batched', - 'keras_saved_out', - 'model_arch', - 2, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3); --- Assert fit has correct class_values -SELECT assert( - dependent_vartype = 'text' AND - class_values = '{NULL,cat,dog,NULL,NULL}', - 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM keras_saved_out_summary) summary; - --- Test with INTEGER class_values --- with NULL as a valid class value -INSERT INTO cifar_10_sample(id, x, y, imgpath) -SELECT 3, x, NULL, '0/img3.jpg' FROM cifar_10_sample -WHERE y = 1; -INSERT INTO cifar_10_sample(id, x, y, imgpath) -SELECT 4, x, 4, '0/img4.jpg' FROM cifar_10_sample -WHERE y = 0; -INSERT INTO cifar_10_sample(id, x, y, imgpath) -SELECT 5, x, 5, '0/img5.jpg' FROM cifar_10_sample -WHERE y = 1; - -DROP TABLE IF EXISTS cifar_10_sample_int_batched; -DROP TABLE IF EXISTS cifar_10_sample_int_batched_summary; -SELECT training_preprocessor_dl('cifar_10_sample','cifar_10_sample_int_batched','y','x', 2, 255, 5); - -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_int_batched', - 'keras_saved_out', - 'model_arch', - 2, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3); - --- Assert fit has correct class_values -SELECT assert( - dependent_vartype = 'smallint' AND - class_values = '{NULL,0,1,4,5}', - 'Keras model output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM keras_saved_out_summary) summary; - -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_test_shape_batched', - 'keras_saved_out', - 'model_arch', - 3, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3); - -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -CREATE TABLE "special-char?" AS SELECT * FROM model_arch; -SELECT madlib_keras_fit( - 'cifar_10_sample_test_shape_batched', - 'keras_saved_out', - '"special-char?"', - 3, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3); diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_gpu_info.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_gpu_info.sql_in deleted file mode 100644 index a8376db..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_gpu_info.sql_in +++ /dev/null @@ -1,42 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -SELECT assert(trap_error($$SELECT * from gpu_configuration('output_table', 'invalid_source')$$) = 1, 'Invalid source'); - -DROP TABLE IF EXISTS output_table_tf; -SELECT gpu_configuration('output_table_tf'); -SELECT * FROM output_table_tf; - -DROP TABLE IF EXISTS output_table_tf; -SELECT gpu_configuration('output_table_tf', NULL); -SELECT * FROM output_table_tf; - -DROP TABLE IF EXISTS output_table_nvidia; -SELECT gpu_configuration('output_table_nvidia','NVIDIA'); -SELECT * FROM output_table_nvidia; - -DROP TABLE IF EXISTS output_table_tf; -SELECT gpu_configuration('output_table_tf', 'tensorflow'); -SELECT * FROM output_table_tf; - -DROP TABLE IF EXISTS output_table_tf; -SELECT * FROM gpu_configuration('output_table_tf'); -SELECT * FROM output_table_tf; \ No newline at end of file diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in deleted file mode 100644 index 389efa3..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_iris.setup.sql_in +++ /dev/null @@ -1,295 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - --------------------- IRIS test input tables ----------------- - -DROP TABLE IF EXISTS iris_data; -CREATE TABLE iris_data( - id serial, - attributes numeric[], - class_text varchar -); -INSERT INTO iris_data(id, attributes, class_text) VALUES -(1,ARRAY[5.1,3.5,1.4,0.2],'Iris-setosa'), -(2,ARRAY[4.9,3.0,1.4,0.2],'Iris-setosa'), -(3,ARRAY[4.7,3.2,1.3,0.2],'Iris-setosa'), -(4,ARRAY[4.6,3.1,1.5,0.2],'Iris-setosa'), -(5,ARRAY[5.0,3.6,1.4,0.2],'Iris-setosa'), -(6,ARRAY[5.4,3.9,1.7,0.4],'Iris-setosa'), -(7,ARRAY[4.6,3.4,1.4,0.3],'Iris-setosa'), -(8,ARRAY[5.0,3.4,1.5,0.2],'Iris-setosa'), -(9,ARRAY[4.4,2.9,1.4,0.2],'Iris-setosa'), -(10,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'), -(11,ARRAY[5.4,3.7,1.5,0.2],'Iris-setosa'), -(12,ARRAY[4.8,3.4,1.6,0.2],'Iris-setosa'), -(13,ARRAY[4.8,3.0,1.4,0.1],'Iris-setosa'), -(14,ARRAY[4.3,3.0,1.1,0.1],'Iris-setosa'), -(15,ARRAY[5.8,4.0,1.2,0.2],'Iris-setosa'), -(16,ARRAY[5.7,4.4,1.5,0.4],'Iris-setosa'), -(17,ARRAY[5.4,3.9,1.3,0.4],'Iris-setosa'), -(18,ARRAY[5.1,3.5,1.4,0.3],'Iris-setosa'), -(19,ARRAY[5.7,3.8,1.7,0.3],'Iris-setosa'), -(20,ARRAY[5.1,3.8,1.5,0.3],'Iris-setosa'), -(21,ARRAY[5.4,3.4,1.7,0.2],'Iris-setosa'), -(22,ARRAY[5.1,3.7,1.5,0.4],'Iris-setosa'), -(23,ARRAY[4.6,3.6,1.0,0.2],'Iris-setosa'), -(24,ARRAY[5.1,3.3,1.7,0.5],'Iris-setosa'), -(25,ARRAY[4.8,3.4,1.9,0.2],'Iris-setosa'), -(26,ARRAY[5.0,3.0,1.6,0.2],'Iris-setosa'), -(27,ARRAY[5.0,3.4,1.6,0.4],'Iris-setosa'), -(28,ARRAY[5.2,3.5,1.5,0.2],'Iris-setosa'), -(29,ARRAY[5.2,3.4,1.4,0.2],'Iris-setosa'), -(30,ARRAY[4.7,3.2,1.6,0.2],'Iris-setosa'), -(31,ARRAY[4.8,3.1,1.6,0.2],'Iris-setosa'), -(32,ARRAY[5.4,3.4,1.5,0.4],'Iris-setosa'), -(33,ARRAY[5.2,4.1,1.5,0.1],'Iris-setosa'), -(34,ARRAY[5.5,4.2,1.4,0.2],'Iris-setosa'), -(35,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'), -(36,ARRAY[5.0,3.2,1.2,0.2],'Iris-setosa'), -(37,ARRAY[5.5,3.5,1.3,0.2],'Iris-setosa'), -(38,ARRAY[4.9,3.1,1.5,0.1],'Iris-setosa'), -(39,ARRAY[4.4,3.0,1.3,0.2],'Iris-setosa'), -(40,ARRAY[5.1,3.4,1.5,0.2],'Iris-setosa'), -(41,ARRAY[5.0,3.5,1.3,0.3],'Iris-setosa'), -(42,ARRAY[4.5,2.3,1.3,0.3],'Iris-setosa'), -(43,ARRAY[4.4,3.2,1.3,0.2],'Iris-setosa'), -(44,ARRAY[5.0,3.5,1.6,0.6],'Iris-setosa'), -(45,ARRAY[5.1,3.8,1.9,0.4],'Iris-setosa'), -(46,ARRAY[4.8,3.0,1.4,0.3],'Iris-setosa'), -(47,ARRAY[5.1,3.8,1.6,0.2],'Iris-setosa'), -(48,ARRAY[4.6,3.2,1.4,0.2],'Iris-setosa'), -(49,ARRAY[5.3,3.7,1.5,0.2],'Iris-setosa'), -(50,ARRAY[5.0,3.3,1.4,0.2],'Iris-setosa'), -(51,ARRAY[7.0,3.2,4.7,1.4],'Iris-versicolor'), -(52,ARRAY[6.4,3.2,4.5,1.5],'Iris-versicolor'), -(53,ARRAY[6.9,3.1,4.9,1.5],'Iris-versicolor'), -(54,ARRAY[5.5,2.3,4.0,1.3],'Iris-versicolor'), -(55,ARRAY[6.5,2.8,4.6,1.5],'Iris-versicolor'), -(56,ARRAY[5.7,2.8,4.5,1.3],'Iris-versicolor'), -(57,ARRAY[6.3,3.3,4.7,1.6],'Iris-versicolor'), -(58,ARRAY[4.9,2.4,3.3,1.0],'Iris-versicolor'), -(59,ARRAY[6.6,2.9,4.6,1.3],'Iris-versicolor'), -(60,ARRAY[5.2,2.7,3.9,1.4],'Iris-versicolor'), -(61,ARRAY[5.0,2.0,3.5,1.0],'Iris-versicolor'), -(62,ARRAY[5.9,3.0,4.2,1.5],'Iris-versicolor'), -(63,ARRAY[6.0,2.2,4.0,1.0],'Iris-versicolor'), -(64,ARRAY[6.1,2.9,4.7,1.4],'Iris-versicolor'), -(65,ARRAY[5.6,2.9,3.6,1.3],'Iris-versicolor'), -(66,ARRAY[6.7,3.1,4.4,1.4],'Iris-versicolor'), -(67,ARRAY[5.6,3.0,4.5,1.5],'Iris-versicolor'), -(68,ARRAY[5.8,2.7,4.1,1.0],'Iris-versicolor'), -(69,ARRAY[6.2,2.2,4.5,1.5],'Iris-versicolor'), -(70,ARRAY[5.6,2.5,3.9,1.1],'Iris-versicolor'), -(71,ARRAY[5.9,3.2,4.8,1.8],'Iris-versicolor'), -(72,ARRAY[6.1,2.8,4.0,1.3],'Iris-versicolor'), -(73,ARRAY[6.3,2.5,4.9,1.5],'Iris-versicolor'), -(74,ARRAY[6.1,2.8,4.7,1.2],'Iris-versicolor'), -(75,ARRAY[6.4,2.9,4.3,1.3],'Iris-versicolor'), -(76,ARRAY[6.6,3.0,4.4,1.4],'Iris-versicolor'), -(77,ARRAY[6.8,2.8,4.8,1.4],'Iris-versicolor'), -(78,ARRAY[6.7,3.0,5.0,1.7],'Iris-versicolor'), -(79,ARRAY[6.0,2.9,4.5,1.5],'Iris-versicolor'), -(80,ARRAY[5.7,2.6,3.5,1.0],'Iris-versicolor'), -(81,ARRAY[5.5,2.4,3.8,1.1],'Iris-versicolor'), -(82,ARRAY[5.5,2.4,3.7,1.0],'Iris-versicolor'), -(83,ARRAY[5.8,2.7,3.9,1.2],'Iris-versicolor'), -(84,ARRAY[6.0,2.7,5.1,1.6],'Iris-versicolor'), -(85,ARRAY[5.4,3.0,4.5,1.5],'Iris-versicolor'), -(86,ARRAY[6.0,3.4,4.5,1.6],'Iris-versicolor'), -(87,ARRAY[6.7,3.1,4.7,1.5],'Iris-versicolor'), -(88,ARRAY[6.3,2.3,4.4,1.3],'Iris-versicolor'), -(89,ARRAY[5.6,3.0,4.1,1.3],'Iris-versicolor'), -(90,ARRAY[5.5,2.5,4.0,1.3],'Iris-versicolor'), -(91,ARRAY[5.5,2.6,4.4,1.2],'Iris-versicolor'), -(92,ARRAY[6.1,3.0,4.6,1.4],'Iris-versicolor'), -(93,ARRAY[5.8,2.6,4.0,1.2],'Iris-versicolor'), -(94,ARRAY[5.0,2.3,3.3,1.0],'Iris-versicolor'), -(95,ARRAY[5.6,2.7,4.2,1.3],'Iris-versicolor'), -(96,ARRAY[5.7,3.0,4.2,1.2],'Iris-versicolor'), -(97,ARRAY[5.7,2.9,4.2,1.3],'Iris-versicolor'), -(98,ARRAY[6.2,2.9,4.3,1.3],'Iris-versicolor'), -(99,ARRAY[5.1,2.5,3.0,1.1],'Iris-versicolor'), -(100,ARRAY[5.7,2.8,4.1,1.3],'Iris-versicolor'), -(101,ARRAY[6.3,3.3,6.0,2.5],'Iris-virginica'), -(102,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'), -(103,ARRAY[7.1,3.0,5.9,2.1],'Iris-virginica'), -(104,ARRAY[6.3,2.9,5.6,1.8],'Iris-virginica'), -(105,ARRAY[6.5,3.0,5.8,2.2],'Iris-virginica'), -(106,ARRAY[7.6,3.0,6.6,2.1],'Iris-virginica'), -(107,ARRAY[4.9,2.5,4.5,1.7],'Iris-virginica'), -(108,ARRAY[7.3,2.9,6.3,1.8],'Iris-virginica'), -(109,ARRAY[6.7,2.5,5.8,1.8],'Iris-virginica'), -(110,ARRAY[7.2,3.6,6.1,2.5],'Iris-virginica'), -(111,ARRAY[6.5,3.2,5.1,2.0],'Iris-virginica'), -(112,ARRAY[6.4,2.7,5.3,1.9],'Iris-virginica'), -(113,ARRAY[6.8,3.0,5.5,2.1],'Iris-virginica'), -(114,ARRAY[5.7,2.5,5.0,2.0],'Iris-virginica'), -(115,ARRAY[5.8,2.8,5.1,2.4],'Iris-virginica'), -(116,ARRAY[6.4,3.2,5.3,2.3],'Iris-virginica'), -(117,ARRAY[6.5,3.0,5.5,1.8],'Iris-virginica'), -(118,ARRAY[7.7,3.8,6.7,2.2],'Iris-virginica'), -(119,ARRAY[7.7,2.6,6.9,2.3],'Iris-virginica'), -(120,ARRAY[6.0,2.2,5.0,1.5],'Iris-virginica'), -(121,ARRAY[6.9,3.2,5.7,2.3],'Iris-virginica'), -(122,ARRAY[5.6,2.8,4.9,2.0],'Iris-virginica'), -(123,ARRAY[7.7,2.8,6.7,2.0],'Iris-virginica'), -(124,ARRAY[6.3,2.7,4.9,1.8],'Iris-virginica'), -(125,ARRAY[6.7,3.3,5.7,2.1],'Iris-virginica'), -(126,ARRAY[7.2,3.2,6.0,1.8],'Iris-virginica'), -(127,ARRAY[6.2,2.8,4.8,1.8],'Iris-virginica'), -(128,ARRAY[6.1,3.0,4.9,1.8],'Iris-virginica'), -(129,ARRAY[6.4,2.8,5.6,2.1],'Iris-virginica'), -(130,ARRAY[7.2,3.0,5.8,1.6],'Iris-virginica'), -(131,ARRAY[7.4,2.8,6.1,1.9],'Iris-virginica'), -(132,ARRAY[7.9,3.8,6.4,2.0],'Iris-virginica'), -(133,ARRAY[6.4,2.8,5.6,2.2],'Iris-virginica'), -(134,ARRAY[6.3,2.8,5.1,1.5],'Iris-virginica'), -(135,ARRAY[6.1,2.6,5.6,1.4],'Iris-virginica'), -(136,ARRAY[7.7,3.0,6.1,2.3],'Iris-virginica'), -(137,ARRAY[6.3,3.4,5.6,2.4],'Iris-virginica'), -(138,ARRAY[6.4,3.1,5.5,1.8],'Iris-virginica'), -(139,ARRAY[6.0,3.0,4.8,1.8],'Iris-virginica'), -(140,ARRAY[6.9,3.1,5.4,2.1],'Iris-virginica'), -(141,ARRAY[6.7,3.1,5.6,2.4],'Iris-virginica'), -(142,ARRAY[6.9,3.1,5.1,2.3],'Iris-virginica'), -(143,ARRAY[5.8,2.7,5.1,1.9],'Iris-virginica'), -(144,ARRAY[6.8,3.2,5.9,2.3],'Iris-virginica'), -(145,ARRAY[6.7,3.3,5.7,2.5],'Iris-virginica'), -(146,ARRAY[6.7,3.0,5.2,2.3],'Iris-virginica'), -(147,ARRAY[6.3,2.5,5.0,1.9],'Iris-virginica'), -(148,ARRAY[6.5,3.0,5.2,2.0],'Iris-virginica'), -(149,ARRAY[6.2,3.4,5.4,2.3],'Iris-virginica'), -(150,ARRAY[5.9,3.0,5.1,1.8],'Iris-virginica'); - - -DROP TABLE IF EXISTS iris_data_one_hot_encoded; -CREATE TABLE iris_data_one_hot_encoded as select id, attributes, ARRAY[class_text is not distinct from 'Iris-setosa', class_text is not distinct from 'Iris-versicolor', class_text is not distinct from 'Iris-virginica']::int[] as class_one_hot_encoded -from iris_data; - - -DROP TABLE IF EXISTS iris_data_packed, iris_data_packed_summary; -SELECT training_preprocessor_dl('iris_data', -- Source table - 'iris_data_packed', -- Output table - 'class_text', -- Dependent variable - 'attributes' -- Independent variable - ); - -DROP TABLE IF EXISTS iris_data_one_hot_encoded_packed, iris_data_one_hot_encoded_packed_summary; -SELECT training_preprocessor_dl('iris_data_one_hot_encoded', -- Source table - 'iris_data_one_hot_encoded_packed', -- Output table - 'class_one_hot_encoded', -- Dependent variable - 'attributes' -- Independent variable - ); - -DROP TABLE IF EXISTS iris_data_val, iris_data_val_summary; -SELECT validation_preprocessor_dl('iris_data', -- Source table - 'iris_data_val', -- Output table - 'class_text', -- Dependent variable - 'attributes', -- Independent variable - 'iris_data_packed'-- Training preprocessed table - ); - -DROP TABLE IF EXISTS iris_data_one_hot_encoded_val, iris_data_one_hot_encoded_val_summary; -SELECT validation_preprocessor_dl('iris_data_one_hot_encoded', -- Source table - 'iris_data_one_hot_encoded_val', -- Output table - 'class_one_hot_encoded', -- Dependent variable - 'attributes', -- Independent variable - 'iris_data_one_hot_encoded_packed' -- Training preprocessed table - ); - -DROP TABLE IF EXISTS iris_model_arch; --- NOTE: The seed is set to 0 for every layer. -SELECT load_keras_model('iris_model_arch', -- Output table, -$$ -{ -"class_name": "Sequential", -"keras_version": "2.1.6", -"config": - [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", - "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}}, - "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, - "bias_constraint": null, "dtype": "float32", "activation": "relu", "trainable": true, - "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", - "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, - "activity_regularizer": null}}, {"class_name": "Dense", - "config": {"kernel_initializer": {"class_name": "VarianceScaling", - "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}}, - "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, - "bias_constraint": null, "activation": "relu", "trainable": true, "kernel_regularizer": null, - "bias_initializer": {"class_name": "Zeros", "config": {}}, "units": 10, "use_bias": true, - "activity_regularizer": null}}, {"class_name": "Dense", "config": {"kernel_initializer": - {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, - "seed": 0, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, - "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", - "trainable": true, "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", - "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], - "backend": "tensorflow"} -$$ -); - --- Duplicate the architecture, but note that trainable is set to FALSE. --- This is to ensure we don't learn anything new, that would help us --- deterministically assert the accuracy and loss after transfer learning --- and warm start. -SELECT load_keras_model('iris_model_arch', -- Output table, -$$ -{ -"class_name": "Sequential", -"keras_version": "2.1.6", -"config": - [{"class_name": "Dense", "config": {"kernel_initializer": {"class_name": "VarianceScaling", - "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}}, - "name": "dense_1", "kernel_constraint": null, "bias_regularizer": null, - "bias_constraint": null, "dtype": "float32", "activation": "relu", - "trainable": false, - "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", - "config": {}}, "units": 10, "batch_input_shape": [null, 4], "use_bias": true, - "activity_regularizer": null}}, {"class_name": "Dense", - "config": {"kernel_initializer": {"class_name": "VarianceScaling", - "config": {"distribution": "uniform", "scale": 1.0, "seed": 0, "mode": "fan_avg"}}, - "name": "dense_2", "kernel_constraint": null, "bias_regularizer": null, - "bias_constraint": null, "activation": "relu", - "trainable": false, - "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", - "config": {}}, "units": 10, "use_bias": true, "activity_regularizer": null}}, - {"class_name": "Dense", "config": {"kernel_initializer": - {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, - "seed": 0, "mode": "fan_avg"}}, "name": "dense_3", "kernel_constraint": null, - "bias_regularizer": null, "bias_constraint": null, "activation": "softmax", - "trainable": false, - "kernel_regularizer": null, "bias_initializer": {"class_name": "Zeros", - "config": {}}, "units": 3, "use_bias": true, "activity_regularizer": null}}], - "backend": "tensorflow"} -$$ -); - -DROP TABLE IF EXISTS iris_train, iris_test; --- Set seed so results are reproducible -SELECT setseed(0); -SELECT train_test_split('iris_data', -- Source table - 'iris', -- Output table root name - 0.8, -- Train proportion - NULL, -- Test proportion (0.2) - NULL, -- Strata definition - NULL, -- Output all columns - NULL, -- Sample without replacement - TRUE -- Separate output tables - ); diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in deleted file mode 100644 index 20e2332..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_averaging_e2e.sql_in +++ /dev/null @@ -1,139 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -m4_include(`SQLCommon.m4') - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in' -) - -DROP TABLE if exists pg_temp.iris_model, pg_temp.iris_model_summary; -SELECT madlib_keras_fit( - 'iris_data_packed', - 'pg_temp.iris_model', - 'iris_model_arch', - 1, - $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$, - $$batch_size=16, epochs=1$$, - 3, - FALSE -); -SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END; - -SELECT assert( - model_arch_table = 'iris_model_arch' AND - validation_table is NULL AND - source_table = 'iris_data_packed' AND - model = 'pg_temp.iris_model' AND - dependent_varname = 'class_text' AND - independent_varname = 'attributes' AND - madlib_version is NOT NULL AND - num_iterations = 3 AND - start_training_time < now() AND - end_training_time < now() AND - num_classes = 3 AND - class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND - dependent_vartype LIKE '%char%' AND - normalizing_const = 1, - 'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM pg_temp.iris_model_summary) summary; - --- Run Predict -DROP TABLE IF EXISTS pg_temp.iris_predict; -SELECT madlib_keras_predict( - 'pg_temp.iris_model', - 'iris_data', - 'id', - 'attributes', - 'pg_temp.iris_predict', - 'prob', - FALSE); -SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END; - --- Run Evaluate -DROP TABLE IF EXISTS pg_temp.evaluate_out; -SELECT madlib_keras_evaluate( - 'pg_temp.iris_model', - 'iris_data_val', - 'pg_temp.evaluate_out', - FALSE); - -SELECT assert(loss >= 0 AND - metric >= 0 AND - metrics_type = '{accuracy}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out)) -FROM pg_temp.evaluate_out; -SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END; - --- Test for one-hot encoded user input data -DROP TABLE if exists iris_model, iris_model_summary, iris_model_info; -SELECT madlib_keras_fit( - 'iris_data_one_hot_encoded_packed', - 'iris_model', - 'iris_model_arch', - 1, - $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$$, - $$batch_size=16, epochs=1$$, - 3, - FALSE -); - -SELECT assert( - model_arch_table = 'iris_model_arch' AND - validation_table is NULL AND - source_table = 'iris_data_one_hot_encoded_packed' AND - model = 'iris_model' AND - dependent_varname = 'class_one_hot_encoded' AND - independent_varname = 'attributes' AND - madlib_version is NOT NULL AND - num_iterations = 3 AND - start_training_time < now() AND - end_training_time < now() AND - dependent_vartype = 'integer[]' AND - num_classes = NULL AND - class_values = NULL AND - normalizing_const = 1, - 'Keras Fit Multiple Output Summary Validation failed when user passes in 1-hot encoded label vector. Actual:' || __to_char(summary)) -FROM (SELECT * FROM iris_model_summary) summary; - --- Run Predict -DROP TABLE IF EXISTS iris_predict; -SELECT madlib_keras_predict( - 'iris_model', - 'iris_data_one_hot_encoded', - 'id', - 'attributes', - 'iris_predict', - 'prob', - FALSE); - --- Run Evaluate -DROP TABLE IF EXISTS evaluate_out; -SELECT madlib_keras_evaluate( - 'iris_model', - 'iris_data_one_hot_encoded_val', - 'evaluate_out', - FALSE); - -SELECT assert(loss >= 0 AND - metric >= 0 AND - metrics_type = '{accuracy}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out)) -FROM evaluate_out; diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in deleted file mode 100644 index e1dbe0c..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in +++ /dev/null @@ -1,426 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -m4_include(`SQLCommon.m4') - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in' -) - --- MST table generation tests --- Valid inputs should pass and yield 6 msts in the table -DROP TABLE IF EXISTS mst_table, mst_table_summary; -SELECT load_model_selection_table( - 'iris_model_arch', - 'mst_table', - ARRAY[1], - ARRAY[ - $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$, - $$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy']$$, - $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$ - ], - ARRAY[ - $$batch_size=5,epochs=1$$, - $$batch_size=10,epochs=1$$ - ] -); -SELECT assert( - COUNT(*)=6, - 'The length of mst table does not match with the inputs' -) -FROM mst_table; - --- Test summary table output -SELECT assert( - model_arch_table = 'iris_model_arch', - 'Model selection output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM mst_table_summary) summary; - --- Invalid arguments must be errored out - -DROP TABLE IF EXISTS mst_table, mst_table_summary; -SELECT assert(trap_error($TRAP$ - SELECT load_model_selection_table( - 'iris_model_arch', - 'mst_table', - ARRAY[-1], - ARRAY[ - $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$ - ], - ARRAY[ - $$batch_size=5,epochs=1$$ - ] - ); -$TRAP$)=1, 'Should error out if model_id is not in the model arch table'); - -DROP TABLE IF EXISTS mst_table, mst_table_summary; -SELECT assert(trap_error($TRAP$ - SELECT load_model_selection_table( - 'iris_model_arch', - 'mst_table', - ARRAY[1], - ARRAY[ - $$foo='bar'$$ - ], - ARRAY[ - $$batch_size='bar'$$ - ] - ); -$TRAP$)=1, 'Should error out if the provided parameters are not valid'); - --- Must deduplicate, options with extra white spaces should not be considered --- as distinct params. - -DROP TABLE IF EXISTS mst_table, mst_table_summary; -SELECT load_model_selection_table( - 'iris_model_arch', - 'mst_table', - ARRAY[1], - ARRAY[ - $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$, - $$ loss='categorical_crossentropy', optimizer='Adam(lr=0.1)',metrics=['accuracy'] $$, - $$loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy']$$ - ], - ARRAY[ - $$batch_size=5,epochs=1$$, - $$batch_size=10,epochs=1$$ - ] -); -SELECT assert( - COUNT(*)=4, - 'The length of mst table (' || COUNT(*) || ')does not match with the inputs due to deduplication failure' -) -FROM mst_table; - --- Must also handle duplicates where order of key/value pairs is re-arranged -DROP TABLE IF EXISTS mst_table, mst_table_summary; -SELECT load_model_selection_table( - 'iris_model_arch', - 'mst_table', - ARRAY[1], - ARRAY[ - $$loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy']$$, - $$metrics= ['accuracy'], loss='categorical_crossentropy', optimizer='Adam(lr=0.1)'$$, - $$loss='mse',optimizer='Adam(lr=0.001)', metrics=['accuracy']$$ - ], - ARRAY[ - $$batch_size=5,epochs=1$$, - $$epochs=1, batch_size=5$$ - ] -); -SELECT assert( - COUNT(*)=2, - 'The length of mst table (' || COUNT(*) || ') does not match with the inputs due to deduplication failure' -) -FROM mst_table; - -m4_changequote(`') -m4_ifdef(, , 0 AND - fit_params = $MAD$batch_size=50, epochs=1$MAD$::text AND - metrics_type = '{accuracy}' AND - training_metrics_final >= 0 AND - training_loss_final >= 0 AND - array_upper(training_metrics, 1) = 1 AND - array_upper(training_loss, 1) = 1 AND - validation_metrics_final >= 0 AND - validation_loss_final >= 0 AND - array_upper(validation_metrics, 1) = 1 AND - array_upper(validation_loss, 1) = 1 AND - array_upper(metrics_elapsed_time, 1) = 1, - 'Keras Fit Multiple Output Info Validation failed. Actual:' || __to_char(info)) -FROM (SELECT * FROM iris_multiple_model_info) info; - -SELECT assert(cnt = 1, - 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info)) -FROM (SELECT count(*) cnt FROM iris_multiple_model_info -WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$MAD$::text) info; - -SELECT assert(cnt = 1, - 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info)) -FROM (SELECT count(*) cnt FROM iris_multiple_model_info -WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.001)', metrics=['accuracy']$MAD$::text) info; - -SELECT assert(cnt = 1, - 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info)) -FROM (SELECT count(*) cnt FROM iris_multiple_model_info -WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.0001)', metrics=['accuracy']$MAD$::text) info; - -SELECT assert( - training_loss[6]-training_loss[1] < 0.1 AND - training_metrics[6]-training_metrics[1] > -0.1, - 'The loss and accuracy should have improved with more iterations.' -) -FROM iris_multiple_model_info -WHERE compile_params like '%lr=0.001%'; - --- Test when number of configs(1) is less than number of segments(3) -DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info; -SELECT madlib_keras_fit_multiple_model( - 'iris_data_packed', - 'iris_multiple_model', - 'mst_table_1row', - 3, - FALSE, - NULL, - 1, - FALSE, - 'multi_model_name', - 'multi_model_descr' -); - -SELECT assert(COUNT(*)=1, 'Info table must have exactly same rows as the number of msts.') -FROM iris_multiple_model_info; - -SELECT assert( - model_id = 1 AND - model_type = 'madlib_keras' AND - model_size > 0 AND - fit_params = $MAD$batch_size=16, epochs=1$MAD$::text AND - metrics_type = '{accuracy}' AND - training_metrics_final >= 0 AND - training_loss_final >= 0 AND - array_upper(training_metrics, 1) = 3 AND - array_upper(training_loss, 1) = 3 AND - array_upper(metrics_elapsed_time, 1) = 3, - 'Keras Fit Multiple Output Info Validation failed. Actual:' || __to_char(info)) -FROM (SELECT * FROM iris_multiple_model_info) info; - -SELECT assert(metrics_elapsed_time[3] - metrics_elapsed_time[1] > 0, - 'Keras Fit Multiple invalid elapsed time calculation.') -FROM (SELECT * FROM iris_multiple_model_info) info; - -SELECT assert( - name = 'multi_model_name' AND - description = 'multi_model_descr' AND - metrics_compute_frequency = 1, - 'Keras Fit Multiple Output Summary Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM iris_multiple_model_summary) summary; - -SELECT assert(cnt = 1, - 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info)) -FROM (SELECT count(*) cnt FROM iris_multiple_model_info -WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$MAD$::text) info; - --- Test when number of configs(4) larger than number of segments(3) -DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info; -SELECT madlib_keras_fit_multiple_model( - 'iris_data_packed', - 'iris_multiple_model', - 'mst_table_4row', - 3, - FALSE -); - --- The default value of the guc 'dev_opt_unsafe_truncate_in_subtransaction' is 'off' --- but we change it to 'on' in fit_multiple.py. Assert that the value is --- reset after calling fit_multiple -SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('dev_opt_unsafe_truncate_in_subtransaction', 'off') END; - -SELECT assert(COUNT(*)=4, 'Info table must have exactly same rows as the number of msts.') -FROM iris_multiple_model_info; - -SELECT assert( - model_id = 1 AND - model_type = 'madlib_keras' AND - model_size > 0 AND - metrics_type = '{accuracy}' AND - training_metrics_final >= 0 AND - training_loss_final >= 0 AND - array_upper(training_metrics, 1) = 1 AND - array_upper(training_loss, 1) = 1 AND - array_upper(metrics_elapsed_time, 1) = 1, - 'Keras Fit Multiple Output Info Validation failed. Actual:' || __to_char(info)) -FROM (SELECT * FROM iris_multiple_model_info) info; - -SELECT assert(cnt = 1, - 'Keras Fit Multiple Output Info compile params validation failed. Actual:' || __to_char(info)) -FROM (SELECT count(*) cnt FROM iris_multiple_model_info -WHERE compile_params = $MAD$loss='categorical_crossentropy', optimizer='Adam(lr=0.01)', metrics=['accuracy']$MAD$::text -AND fit_params = $MAD$batch_size=32, epochs=1$MAD$::text) info; - --- Test when class values have NULL values -UPDATE iris_data_packed_summary SET class_values = ARRAY['Iris-setosa','Iris-versicolor',NULL]; -DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info; -SELECT madlib_keras_fit_multiple_model( - 'iris_data_packed', - 'iris_multiple_model', - 'mst_table_1row', - 1, - FALSE, - NULL, - 1, - FALSE -); - -SELECT assert( - num_classes = 3 AND - class_values = '{Iris-setosa,Iris-versicolor,NULL}', - 'Keras Fit Multiple num_clases and class values Validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM iris_multiple_model_summary) summary; - -------------- Test for schema qualified output table and input table ----------------------- -CREATE SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__; -CREATE TABLE __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_data_packed as select * from iris_data_packed; -CREATE TABLE __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_data_packed_summary as select * from iris_data_packed_summary; - --- do not drop the output table created in the previous test -SELECT count(*) from iris_multiple_model; -SELECT madlib_keras_fit_multiple_model( - '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_data_packed', - '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model', - 'mst_table_1row', - 1, - FALSE, - NULL, - 1, - FALSE -); -SELECT count(*) from __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model; -SELECT assert( - num_classes = 3 AND - class_values = '{Iris-setosa,Iris-versicolor,NULL}', - 'Keras Fit Multiple validation failed. Actual:' || __to_char(summary)) -FROM (SELECT * FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_multiple_model_summary) summary; - -DROP SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__ CASCADE; -!>) diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in deleted file mode 100644 index d738f48..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection_e2e.sql_in +++ /dev/null @@ -1,165 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -m4_include(`SQLCommon.m4') - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in' -) - -m4_changequote(`') -m4_ifdef(, , = 0 AND - metric >= 0 AND - metrics_type = '{accuracy}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out)) -FROM pg_temp.evaluate_out; - --- Test for one-hot encoded user input data -DROP TABLE if exists iris_multiple_model, iris_multiple_model_summary, iris_multiple_model_info; -SELECT madlib_keras_fit_multiple_model( - 'iris_data_one_hot_encoded_packed', - 'iris_multiple_model', - 'pg_temp.mst_table', - 3, - FALSE -); - -SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END; - -SELECT assert( - model_arch_table = 'iris_model_arch' AND - validation_table is NULL AND - model_info = 'iris_multiple_model_info' AND - source_table = 'iris_data_one_hot_encoded_packed' AND - model = 'iris_multiple_model' AND - dependent_varname = 'class_one_hot_encoded' AND - independent_varname = 'attributes' AND - madlib_version is NOT NULL AND - num_iterations = 3 AND - start_training_time < now() AND - end_training_time < now() AND - dependent_vartype = 'integer[]' AND - num_classes = NULL AND - class_values = NULL AND - normalizing_const = 1, - 'Keras Fit Multiple Output Summary Validation failed when user passes in 1-hot encoded label vector. Actual:' || __to_char(summary)) -FROM (SELECT * FROM iris_multiple_model_summary) summary; - --- Run Predict -DROP TABLE IF EXISTS iris_predict; -SELECT madlib_keras_predict( - 'iris_multiple_model', - 'iris_data_one_hot_encoded', - 'id', - 'attributes', - 'iris_predict', - 'prob', - NULL, - 1); -SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END; - --- Run Evaluate -DROP TABLE IF EXISTS evaluate_out; -SELECT madlib_keras_evaluate( - 'iris_multiple_model', - 'iris_data_one_hot_encoded_val', - 'evaluate_out', - NULL, - 1); -SELECT CASE WHEN is_ver_greater_than_gp_640_or_pg_11() is TRUE THEN assert_guc_value('plan_cache_mode', 'auto') END; - -SELECT assert(loss >= 0 AND - metric >= 0 AND - metrics_type = '{accuracy}', 'Evaluate output validation failed. Actual:' || __to_char(evaluate_out)) -FROM evaluate_out; - -DROP SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__ CASCADE; -!>) diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in deleted file mode 100644 index 5a5b362..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict.sql_in +++ /dev/null @@ -1,403 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -m4_include(`SQLCommon.m4') - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_cifar.setup.sql_in' -) - --- Please do not break up the compile_params string --- It might break the assertion -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_batched', - 'keras_saved_out', - 'model_arch', - 1, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3); - -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample', - 'id', - 'x', - 'cifar10_predict', - NULL, - FALSE); - --- Validate that prediction output table exists and has correct schema -SELECT assert(UPPER(pg_typeof(id)::TEXT) = 'INTEGER', 'id column should be INTEGER type') - FROM cifar10_predict; - -SELECT assert(UPPER(pg_typeof(estimated_y)::TEXT) = - 'SMALLINT', 'prediction column should be SMALLINT type') - FROM cifar10_predict; - --- Validate correct number of rows returned. -SELECT assert(COUNT(*)=2, 'Output table of madlib_keras_predict should have two rows') -FROM cifar10_predict; - --- First test that all values are in set of class values; if this breaks, it's definitely a problem. -SELECT assert(estimated_y IN (0,1), - 'Predicted value not in set of defined class values for model') -FROM cifar10_predict; - -DROP TABLE IF EXISTS cifar10_predict; -SELECT assert(trap_error($TRAP$SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample_batched', - 'buffer_id', - 'independent_var', - 'cifar10_predict', - NULL, - FALSE);$TRAP$) = 1, - 'Passing batched image table to predict should error out.'); - --- Test with pred_type=prob -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample', - 'id', - 'x', - 'cifar10_predict', - 'prob', - FALSE); - -SELECT assert(UPPER(pg_typeof(prob_0)::TEXT) = - 'DOUBLE PRECISION', 'column prob_0 should be double precision type') - FROM cifar10_predict; - -SELECT assert(UPPER(pg_typeof(prob_1)::TEXT) = - 'DOUBLE PRECISION', 'column prob_1 should be double precision type') - FROM cifar10_predict; - -SELECT assert(COUNT(*)=3, 'Predict out table must have exactly three cols.') -FROM pg_attribute -WHERE attrelid='cifar10_predict'::regclass AND attnum>0; - --- Tests with text class values: --- Create a new table using the text based column for dep var. -DROP TABLE IF EXISTS cifar_10_sample_text_batched; -m4_changequote(`') -CREATE TABLE cifar_10_sample_text_batched AS - SELECT buffer_id, independent_var, dependent_var, - independent_var_shape, dependent_var_shape - m4_ifdef(, , ) - FROM cifar_10_sample_batched m4_ifdef(, , ); - --- Insert a new row with NULL as the dependent var (one-hot encoded) -UPDATE cifar_10_sample_text_batched - SET dependent_var = convert_array_to_bytea(ARRAY[0,0,1,0,0]::smallint[]) WHERE buffer_id=0; -UPDATE cifar_10_sample_text_batched - SET dependent_var = convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) WHERE buffer_id=1; -INSERT INTO cifar_10_sample_text_batched(m4_ifdef(, , ) buffer_id, independent_var, dependent_var, independent_var_shape, dependent_var_shape) - SELECT m4_ifdef(, , ) 2 AS buffer_id, independent_var, - convert_array_to_bytea(ARRAY[0,1,0,0,0]::smallint[]) AS dependent_var, - independent_var_shape, dependent_var_shape - FROM cifar_10_sample_batched WHERE cifar_10_sample_batched.buffer_id=0; -UPDATE cifar_10_sample_text_batched SET dependent_var_shape = ARRAY[1,5]; -m4_changequote(,) - --- Create the necessary summary table for the batched input. -DROP TABLE IF EXISTS cifar_10_sample_text_batched_summary; -CREATE TABLE cifar_10_sample_text_batched_summary( - source_table text, - output_table text, - dependent_varname text, - independent_varname text, - dependent_vartype text, - class_values text[], - buffer_size integer, - normalizing_const numeric); -INSERT INTO cifar_10_sample_text_batched_summary values ( - 'cifar_10_sample', - 'cifar_10_sample_text_batched', - 'y_text', - 'x', - 'text', - ARRAY[NULL,'cat','dog',NULL,NULL], - 1, - 255.0); - -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_text_batched', - 'keras_saved_out', - 'model_arch', - 2, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3); - --- Predict with pred_type=prob -DROP TABLE IF EXISTS cifar_10_sample_text; -CREATE TABLE cifar_10_sample_text AS - SELECT id, x, y_text - FROM cifar_10_sample; -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample_text', - 'id', - 'x', - 'cifar10_predict', - 'prob', - FALSE); - --- Validate the output datatype of newly created prediction columns --- for prediction type = 'prob' and class_values 'TEXT' with NULL as a valid --- class_values -SELECT assert(UPPER(pg_typeof(prob_cat)::TEXT) = - 'DOUBLE PRECISION', 'column prob_cat should be double precision type') -FROM cifar10_predict; - -SELECT assert(UPPER(pg_typeof(prob_dog)::TEXT) = - 'DOUBLE PRECISION', 'column prob_dog should be double precision type') -FROM cifar10_predict; - -SELECT assert(UPPER(pg_typeof("prob_NULL")::TEXT) = - 'DOUBLE PRECISION', 'column prob_NULL should be double precision type') -FROM cifar10_predict; - --- Must have exactly 4 cols (3 for class_values and 1 for id) -SELECT assert(COUNT(*)=4, 'Predict out table must have exactly four cols.') -FROM pg_attribute -WHERE attrelid='cifar10_predict'::regclass AND attnum>0; - --- Predict with pred_type=response -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample_text', - 'id', - 'x', - 'cifar10_predict', - 'response', - FALSE); - --- Validate the output datatype of newly created prediction columns --- for prediction type = 'response' and class_values 'TEXT' with NULL --- as a valid class_values -SELECT assert(UPPER(pg_typeof(estimated_y_text)::TEXT) = - 'TEXT', 'prediction column should be TEXT type') -FROM cifar10_predict LIMIT 1; - --- Tests where the assumption is user has one-hot encoded, so class_values --- in input summary table will be NULL. -UPDATE keras_saved_out_summary SET class_values=NULL; - --- Predict with pred_type=prob -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample_text', - 'id', - 'x', - 'cifar10_predict', - 'prob', - FALSE); - --- Validate the output datatype of newly created prediction column --- for prediction type = 'response' and class_value = NULL --- Returns: Array of probabilities for user's one-hot encoded data -SELECT assert(UPPER(pg_typeof(prob)::TEXT) = - 'DOUBLE PRECISION[]', 'column prob should be double precision[] type') -FROM cifar10_predict LIMIT 1; - --- Predict with pred_type=response -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample_text', - 'id', - 'x', - 'cifar10_predict', - 'response', - FALSE); - --- Validate the output datatype of newly created prediction column --- for prediction type = 'response' and class_value = NULL --- Returns: Index of class value in user's one-hot encoded data with --- highest probability -SELECT assert(UPPER(pg_typeof(estimated_y_text)::TEXT) = - 'TEXT', 'column estimated_y_text should be text type') -FROM cifar10_predict LIMIT 1; - --- Test predict with INTEGER class_values --- with NULL as a valid class value --- Update output_summary table to reflect --- class_values {NULL,0,1,4,5} and dependent_vartype is SMALLINT -UPDATE keras_saved_out_summary -SET dependent_varname = 'y', - class_values = ARRAY[NULL,0,1,4,5]::INTEGER[], - dependent_vartype = 'smallint'; --- Predict with pred_type=prob -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample', - 'id', - 'x', - 'cifar10_predict', - 'prob', - FALSE); - --- Validate the output datatype of newly created prediction column --- for prediction type = 'prob' and class_values 'INT' with NULL --- as a valid class_values -SELECT assert(UPPER(pg_typeof("prob_NULL")::TEXT) = - 'DOUBLE PRECISION', 'column prob_NULL should be double precision type') -FROM cifar10_predict; - --- Must have exactly 6 cols (5 for class_values and 1 for id) -SELECT assert(COUNT(*)=6, 'Predict out table must have exactly six cols.') -FROM pg_attribute -WHERE attrelid='cifar10_predict'::regclass AND attnum>0; - --- Predict with pred_type=response -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample', - 'id', - 'x', - 'cifar10_predict', - 'response', - FALSE); - --- Validate the output datatype of newly created prediction column --- for prediction type = 'response' and class_values 'TEXT' with NULL --- as a valid class_values --- Returns: class_value with highest probability -SELECT assert(UPPER(pg_typeof(estimated_y)::TEXT) = - 'SMALLINT', 'prediction column should be smallint type') -FROM cifar10_predict; - --- Predict with correctly shaped data, must go thru. --- Update output_summary table to reflect --- class_values, num_classes and model_id for shaped data -DROP TABLE IF EXISTS keras_saved_out, keras_saved_out_summary; -SELECT madlib_keras_fit( - 'cifar_10_sample_test_shape_batched', - 'keras_saved_out', - 'model_arch', - 3, - $$ optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']$$::text, - $$ batch_size=2, epochs=1, verbose=0 $$::text, - 3); - -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample_test_shape', - 'id', - 'x', - 'cifar10_predict', - 'prob', - FALSE); - --- Prediction with incorrectly shaped data must error out. -DROP TABLE IF EXISTS cifar10_predict; -SELECT assert(trap_error($TRAP$SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample', - 'id', - 'x', - 'cifar10_predict', - 'prob', - FALSE);$TRAP$) = 1, - 'Input shape is (32, 32, 3) but model was trained with (3, 32, 32). Should have failed.'); - --- Test model_arch is retrieved from model data table and not model architecture -DROP TABLE IF EXISTS model_arch; -DROP TABLE IF EXISTS cifar10_predict; -SELECT madlib_keras_predict( - 'keras_saved_out', - 'cifar_10_sample_test_shape', - 'id', - 'x', - 'cifar10_predict', - 'prob', - FALSE); - --- Test multi model - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in' -) - -m4_changequote(`') -m4_ifdef(, , ) diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in deleted file mode 100644 index 12dee6f..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_predict_byom.sql_in +++ /dev/null @@ -1,137 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in' -) - -DROP TABLE IF EXISTS iris_model, iris_model_summary; -SELECT madlib_keras_fit('iris_data_packed', -- source table - 'iris_model', -- model output table - 'iris_model_arch', -- model arch table - 1, -- model arch id - $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params - $$ batch_size=5, epochs=3 $$, -- fit_params - 3); -- num_iterations - -DROP TABLE IF EXISTS iris_predict; -SELECT madlib_keras_predict('iris_model', -- model - 'iris_test', -- test_table - 'id', -- id column - 'attributes', -- independent var - 'iris_predict' -- output table - ); - --- Copy weights that were learnt from the previous run, for transfer --- learning. Copy it now, because using warm_start will overwrite it. -UPDATE iris_model_arch set model_weights = (select model_weights from iris_model) WHERE model_id = 2; - --- class_values not NULL, pred_type is response -DROP TABLE IF EXISTS iris_predict_byom; -SELECT madlib_keras_predict_byom( - 'iris_model_arch', - 2, - 'iris_test', - 'id', - 'attributes', - 'iris_predict_byom', - 'response', - NULL, - ARRAY['Iris-setosa', 'Iris-versicolor', - 'Iris-virginica'] - ); - -SELECT assert( - p0.estimated_class_text = p1.estimated_dependent_var, - 'Predict byom failure for non null class value and response pred_type.') -FROM iris_predict AS p0, iris_predict_byom AS p1 -WHERE p0.id=p1.id; -SELECT assert(UPPER(pg_typeof(estimated_dependent_var)::TEXT) = 'TEXT', - 'Predict byom failure for non null class value and response pred_type. - Expeceted estimated_dependent_var to be of type TEXT') -FROM iris_predict_byom LIMIT 1; - --- class_values NULL, pred_type is NULL (response) -DROP TABLE IF EXISTS iris_predict_byom; -SELECT madlib_keras_predict_byom( - 'iris_model_arch', - 2, - 'iris_test', - 'id', - 'attributes', - 'iris_predict_byom' - ); -SELECT assert( - p1.estimated_dependent_var IN ('0', '1', '2'), - 'Predict byom failure for null class value and null pred_type.') -FROM iris_predict_byom AS p1; -SELECT assert(UPPER(pg_typeof(estimated_dependent_var)::TEXT) = 'TEXT', - 'Predict byom failure for non null class value and response pred_type. - Expeceted estimated_dependent_var to be of type TEXT') -FROM iris_predict_byom LIMIT 1; - --- class_values not NULL, pred_type is prob -DROP TABLE IF EXISTS iris_predict_byom; -SELECT madlib_keras_predict_byom( - 'iris_model_arch', - 2, - 'iris_test', - 'id', - 'attributes', - 'iris_predict_byom', - 'prob', - NULL, - ARRAY['Iris-setosa', 'Iris-versicolor', - 'Iris-virginica'], - 1.0 - ); - -SELECT assert( - (p1."prob_Iris-setosa" + p1."prob_Iris-virginica" + p1."prob_Iris-versicolor") - 1 < 1e-6, - 'Predict byom failure for non null class value and prob pred_type.') -FROM iris_predict_byom AS p1; -SELECT assert(UPPER(pg_typeof("prob_Iris-setosa")::TEXT) = 'DOUBLE PRECISION', - 'Predict byom failure for non null class value and prob pred_type. - Expeceted "prob_Iris-setosa" to be of type DOUBLE PRECISION') -FROM iris_predict_byom LIMIT 1; - --- class_values NULL, pred_type is prob -DROP TABLE IF EXISTS iris_predict_byom; -SELECT madlib_keras_predict_byom( - 'iris_model_arch', - 2, - 'iris_test', - 'id', - 'attributes', - 'iris_predict_byom', - 'prob', - NULL, - NULL - ); -SELECT assert( - (prob[1] + prob[2] + prob[3]) - 1 < 1e-6, - 'Predict byom failure for null class value and prob pred_type.') -FROM iris_predict_byom; -SELECT assert(UPPER(pg_typeof(prob)::TEXT) = 'DOUBLE PRECISION[]', - 'Predict byom failure for null class value and prob pred_type. Expeceted prob to - be of type DOUBLE PRECISION[]') -FROM iris_predict_byom LIMIT 1; diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in deleted file mode 100644 index 43ad38b..0000000 --- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_transfer_learning.sql_in +++ /dev/null @@ -1,355 +0,0 @@ -/* ---------------------------------------------------------------------*//** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - *//* ---------------------------------------------------------------------*/ - -m4_include(`SQLCommon.m4') - -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in' -) --------------- Warm start test (along with schema qualified output table) ------------------------- -CREATE SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__; -DROP TABLE IF EXISTS iris_model, iris_model_summary; -SELECT madlib_keras_fit('iris_data_packed', -- source table - '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model', -- model output table - 'iris_model_arch', -- model arch table - 1, -- model arch id - $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params - $$ batch_size=5, epochs=3 $$, -- fit_params - 5, -- num_iterations - NULL, NULL, - 1 -- metrics_compute_frequency - ); - --- Test that our code is indeed learning something and not broken. The loss --- from the first iteration should be less than the 5th, while the accuracy --- must be greater. -SELECT assert( - array_upper(training_loss, 1) = 5 AND - array_upper(training_metrics, 1) = 5, - 'metrics compute frequency must be 1.') -FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary; - -SELECT assert( - training_loss[5]-training_loss[1] < 0.1 AND - training_metrics[5]-training_metrics[1] > -0.1, - 'The loss and accuracy should have improved with more iterations.' -) -FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary; - --- Make a copy of the loss and metrics array, to compare it with runs after --- warm start and transfer learning. -DROP TABLE IF EXISTS iris_model_first_run; -CREATE TABLE iris_model_first_run AS -SELECT training_loss_final, training_metrics_final -FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary; - --- Copy weights that were learnt from the previous run, for transfer --- learning. Copy it now, because using warm_start will overwrite it. -UPDATE iris_model_arch set model_weights = (select model_weights from __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model) -WHERE model_id = 2; - --- Warm start test -SELECT madlib_keras_fit('iris_data_packed', -- source table - '__MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model', -- model output table - 'iris_model_arch', -- model arch table - 2, -- model arch id - $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params - $$ batch_size=5, epochs=3 $$, -- fit_params - 2, -- num_iterations, - NULL, NULL, 1, - TRUE -- warm start - ); - -SELECT assert( - array_upper(training_loss, 1) = 2 AND - array_upper(training_metrics, 1) = 2, - 'metrics compute frequency must be 1.') -FROM __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary; - -SELECT assert( - abs(first.training_loss_final-second.training_loss[1]) < 1e-6 AND - abs(first.training_loss_final-second.training_loss[2]) < 1e-6 AND - abs(first.training_metrics_final-second.training_metrics[1]) < 1e-10 AND - abs(first.training_metrics_final-second.training_metrics[2]) < 1e-10, - 'warm start test failed because training loss and metrics don''t match the expected value from the previous run of keras fit.') -FROM iris_model_first_run AS first, __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__.iris_model_summary AS second; - -DROP SCHEMA __MADLIB__DEEP_LEARNING_SCHEMA__MADLIB__ CASCADE; - ----------------- end Warm start test ---------------------------------------------------- - ----------------- Transfer learning test ---------------------------------------------------- -DROP TABLE IF EXISTS iris_model_transfer, iris_model_transfer_summary; -SELECT madlib_keras_fit('iris_data_packed', -- source table - 'iris_model_transfer', -- model output table - 'iris_model_arch', -- model arch table - 2, -- model arch id - $$ loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] $$, -- compile_params - $$ batch_size=5, epochs=3 $$, -- fit_params - 2, - NULL, NULL, 1 - ); - -SELECT assert( - array_upper(training_loss, 1) = 2 AND - array_upper(training_metrics, 1) = 2, - 'metrics compute frequency must be 1.') -FROM iris_model_transfer_summary; - -SELECT assert( - abs(first.training_loss_final-second.training_loss[1]) < 1e-6 AND - abs(first.training_loss_final-second.training_loss[2]) < 1e-6 AND - abs(first.training_metrics_final-second.training_metrics[1]) < 1e-10 AND - abs(first.training_metrics_final-second.training_metrics[2]) < 1e-10, - 'Transfer learning test failed because training loss and metrics don''t match the expected value.') -FROM iris_model_first_run AS first, -iris_model_transfer_summary AS second; - --- Rerun the iris setup to discard the changes -\i m4_regexp(MODULE_PATHNAME, - `\(.*\)libmadlib\.so', - `\1../../modules/deep_learning/test/madlib_keras_iris.setup.sql_in' -) - -m4_changequote(`') -m4_ifdef(, , 1e-6, - 'Transfer learning test failed because training loss and metrics don''t match the expected value.') -FROM iris_model_first_run AS first, iris_multiple_model_info AS second -WHERE first.mst_key = second.mst_key AND first.model_id = 1; - -!>) - diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/plpy_mock.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/plpy_mock.py_in deleted file mode 100644 index dd18649..0000000 --- a/src/ports/postgres/modules/deep_learning/test/unit_tests/plpy_mock.py_in +++ /dev/null @@ -1,43 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -m4_changequote(`') -def __init__(self): - pass - -def error(message): - raise PLPYException(message) - -def execute(query): - pass - -def warning(query): - pass - -def info(query): - print query - - -class PLPYException(Exception): - def __init__(self, message): - super(PLPYException, self).__init__() - self.message = message - - def __str__(self): - return repr(self.message) diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in deleted file mode 100644 index d2e14cd..0000000 --- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_input_data_preprocessor.py_in +++ /dev/null @@ -1,383 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import sys -import numpy as np -from os import path - -# Add modules to the pythonpath. -sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))) -sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))) - -import unittest -from mock import * -import plpy_mock as plpy - -m4_changequote(`') - -class InputPreProcessorDLTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy, - 'utilities.mean_std_dev_calculator': Mock(), - } - # we need to use MagicMock() instead of Mock() for the plpy.execute mock - # to be able to iterate on the return value - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - - self.default_schema_madlib = "madlib" - self.default_source_table = "source" - self.default_output_table = "output" - self.default_dep_var = "depvar" - self.default_ind_var = "indvar" - self.default_buffer_size = 5 - self.default_normalizing_const = 1.0 - self.default_num_classes = None - self.default_distribution_rules = "all_segments" - self.default_module_name = "dummy" - - import deep_learning.input_data_preprocessor - self.module = deep_learning.input_data_preprocessor - import utilities.minibatch_preprocessing - self.util_module = utilities.minibatch_preprocessing - import utilities.control - self.control_module = utilities.control - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - self.module.validate_module_input_params = Mock() - self.module.get_distinct_col_levels = Mock(return_value = [0,22,100]) - - def tearDown(self): - self.module_patcher.stop() - - def test_input_preprocessor_dl_executes_query(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - self.control_module.OptimizerControl.__enter__ = Mock() - self.control_module.OptimizerControl.optimizer_control = True - self.control_module.OptimizerControl.optimizer_enabled = True - preprocessor_obj = self.module.InputDataPreprocessorDL( - self.default_schema_madlib, - "input", - "out", - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - preprocessor_obj.input_preprocessor_dl() - - def test_input_preprocessor_null_buffer_size_executes_query(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - self.control_module.OptimizerControl.__enter__ = Mock() - self.control_module.OptimizerControl.optimizer_control = True - self.control_module.OptimizerControl.optimizer_enabled = True - preprocessor_obj = self.module.InputDataPreprocessorDL( - self.default_schema_madlib, - "input", - "out", - self.default_dep_var, - self.default_ind_var, - None, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - self.util_module.MiniBatchBufferSizeCalculator.calculate_default_buffer_size = Mock(return_value = 5) - preprocessor_obj.input_preprocessor_dl() - - def test_input_preprocessor_multiple_dep_var_raises_exception(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - with self.assertRaises(plpy.PLPYException): - self.module.InputDataPreprocessorDL( - self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - "y1,y2", - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - - def test_input_preprocessor_multiple_indep_var_raises_exception(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - with self.assertRaises(plpy.PLPYException): - self.module.InputDataPreprocessorDL( - self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - "x1,x2", - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - - def test_input_preprocessor_buffer_size_zero_fails(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - with self.assertRaises(plpy.PLPYException): - self.module.InputDataPreprocessorDL( - self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - 0, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - - def test_input_preprocessor_negative_buffer_size_fails(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - with self.assertRaises(plpy.PLPYException): - self.module.InputDataPreprocessorDL(self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - -1, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - - def test_input_preprocessor_invalid_indep_vartype_raises_exception(self): - self.module.get_expr_type = Mock(side_effect = ['integer', 'integer[]']) - with self.assertRaises(plpy.PLPYException): - self.module.InputDataPreprocessorDL(self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - - def test_input_preprocessor_invalid_dep_vartype_raises_exception(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text[]']) - with self.assertRaises(plpy.PLPYException): - self.module.InputDataPreprocessorDL(self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - - def test_input_preprocessor_normalizing_const_zero_fails(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - with self.assertRaises(plpy.PLPYException): - self.module.InputDataPreprocessorDL(self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - 0, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - - def test_input_preprocessor_negative_normalizing_const_fails(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - with self.assertRaises(plpy.PLPYException): - self.module.InputDataPreprocessorDL(self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - -1, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - - def test_get_one_hot_encoded_dep_var_expr_null_val(self): - self.module.get_expr_type = Mock(side_effect = ['smallint[]', 'text']) - self.module.get_distinct_col_levels = Mock(return_value = ["NULL", "'a'"]) - obj = self.module.InputDataPreprocessorDL( - self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - obj.dependent_levels = ["NULL", "'a'"] - dep_var_array_expr = obj.get_one_hot_encoded_dep_var_expr() - self.assertEqual("array[({0}) is not distinct from null, " \ - "({0}) is not distinct from 'a']::integer[]::smallint[]". - format(self.default_dep_var), - dep_var_array_expr.lower()) - - def test_get_one_hot_encoded_dep_var_expr_numeric_array_val(self): - self.module.get_expr_type = Mock(side_effect = ['smallint[]', 'integer[]']) - obj = self.module.InputDataPreprocessorDL( - self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - dep_var_array_expr = obj.get_one_hot_encoded_dep_var_expr() - self.assertEqual("{0}::smallint[]". - format(self.default_dep_var), - dep_var_array_expr.lower()) - - def test_validate_num_classes_none(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text']) - obj = self.module.InputDataPreprocessorDL( - self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - None, - self.default_distribution_rules, - self.default_module_name) - obj.dependent_levels = ["dummy"] - self.assertEqual(0, obj.padding_size) - - def test_validate_num_classes_greater(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text']) - self.module._get_dependent_levels = Mock(return_value = ["'a'", "'b'", "'c'"]) - obj = self.module.TrainingDataPreprocessorDL( - self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - 5, - self.default_distribution_rules) - obj._set_one_hot_encoding_variables() - self.assertEqual(2, obj.padding_size) - - def test_validate_num_classes_lesser(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'text']) - self.module.dependent_levels = Mock(return_value = ["'a'", "'b'", "'c'"]) - obj = self.module.TrainingDataPreprocessorDL( - self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - 2, - self.default_distribution_rules) - with self.assertRaises(plpy.PLPYException): - obj._set_one_hot_encoding_variables() - - def test_validate_distribution_table(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - - obj = self.module.InputDataPreprocessorDL(self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - self.module.input_tbl_valid = Mock() - self.module.is_var_valid = Mock() - self.plpy_mock_execute.side_effect = [ - [{'dbids': [2,3,4]}], - [{'dbids': [3,4], 'c1': 2, 'c2': 2}] - ] - - obj._validate_distribution_table() - - def test_validate_distribution_table_dup(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - - obj = self.module.InputDataPreprocessorDL(self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - self.module.input_tbl_valid = Mock() - self.module.is_var_valid = Mock() - self.plpy_mock_execute.side_effect = [ - [{'dbids': [2,3,4]}], - [{'dbids': [3,3], 'c1': 2, 'c2': 1}] - ] - with self.assertRaises(plpy.PLPYException) as error: - obj._validate_distribution_table() - self.assertIn('duplicate', str(error.exception).lower()) - - def test_validate_distribution_table_invalid(self): - self.module.get_expr_type = Mock(side_effect = ['integer[]', 'integer[]']) - obj = self.module.InputDataPreprocessorDL(self.default_schema_madlib, - self.default_source_table, - self.default_output_table, - self.default_dep_var, - self.default_ind_var, - self.default_buffer_size, - self.default_normalizing_const, - self.default_num_classes, - self.default_distribution_rules, - self.default_module_name) - self.module.input_tbl_valid = Mock() - self.module.is_var_valid = Mock() - self.plpy_mock_execute.side_effect = [ - [{'dbids': [2,3,4]}], - [{'dbids': [3,30], 'c1': 2, 'c2': 2}] - ] - with self.assertRaises(plpy.PLPYException) as error: - obj._validate_distribution_table() - self.assertIn('invalid', str(error.exception).lower()) - - -if __name__ == '__main__': - unittest.main() - -# --------------------------------------------------------------------- diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in deleted file mode 100644 index ed3e0da..0000000 --- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in +++ /dev/null @@ -1,1681 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -m4_changequote(`') - -import sys -import numpy as np -import os -from os import path -# Add convex module to the pythonpath. -sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))) -sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))) - -import keras -from keras.models import * -from keras.layers import * -import unittest -from mock import * -import plpy_mock as plpy - -# helper for multiplying array by int -def mult(k,arr): - return [ k*a for a in arr ] - -class MadlibKerasFitTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import madlib_keras - self.subject = madlib_keras - - self.model = Sequential() - self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', - input_shape=(1,1,1,), padding='same')) - self.model.add(Flatten()) - - self.compile_params = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']" - self.fit_params = "batch_size=1, epochs=1" - self.model_weights = [3,4,5,6] - self.serialized_weights = np.array(self.model_weights, dtype=np.float32 - ).tostring() - self.dist_key_mapping = [0,1,2] - self.accessible_gpus_for_seg = [0] - - self.independent_var_real = [[[[0.5]]]] * 10 - self.dependent_var_int = [[0,1]] * 10 - - # Params as bytea - self.independent_var = np.array(self.independent_var_real, dtype=np.float32).tobytes() - self.dependent_var = np.array(self.dependent_var_int, dtype=np.int16).tobytes() - - self.independent_var_shape = [10,1,1,1] - self.dependent_var_shape = [10,2] - # We test on segment 0, which has 3 buffers filled with 10 identical - # images each, or 30 images total - self.total_images_per_seg = [3*len(self.dependent_var_int),20,40] - - self.dummy_prev_weights = 'dummy weights' - - def tearDown(self): - self.module_patcher.stop() - - def _test_fit_transition_first_buffer_pass(self, is_platform_pg): - #TODO should we mock tensorflow's close_session and keras' - # clear_session instead of mocking the function `K.clear_session` - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - self.subject.is_platform_pg = Mock(return_value = is_platform_pg) - starting_image_count = 0 - ending_image_count = len(self.dependent_var_int) - - # last iteration Call - previous_state = np.array(self.model_weights, dtype=np.float32) - - k = {'SD': {}} - - new_state = self.subject.fit_transition( - None, self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), self.compile_params, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, previous_state.tostring(), True, **k) - - image_count = new_state - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session and sess.close must not get called for the first buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - self.assertTrue(k['SD']['segment_model']) - - # Non-last iteration Call - self.subject.K.set_session.reset_mock() - self.subject.K.clear_session.reset_mock() - previous_state = np.array(self.model_weights, dtype=np.float32) - - k = {'SD' : {}} - - new_state = self.subject.fit_transition( - None, self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), self.compile_params, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, previous_state.tostring(), False, **k) - - image_count = new_state - - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session and sess.close must not get called for the first buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - self.assertTrue(k['SD']['segment_model']) - - def test_fit_transition_multiple_model_first_buffer_pass(self): - #TODO should we mock tensorflow's close_session and keras' - # clear_session instead of mocking the function `K.clear_session` - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - starting_image_count = 0 - ending_image_count = len(self.dependent_var_int) - - previous_weights = np.array(self.model_weights, dtype=np.float32) - - k = {'SD': {}} - - new_state = self.subject.fit_transition( - None, self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), self.compile_params, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, previous_weights.tostring(), True, True, **k) - - image_count = new_state - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session must not be called for the first buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - self.assertTrue(k['SD']['segment_model']) - - def _test_fit_transition_middle_buffer_pass(self, is_platform_pg): - #TODO should we mock tensorflow's close_session and keras' - # clear_session instead of mocking the function `K.clear_session` - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - self.subject.is_platform_pg = Mock(return_value = is_platform_pg) - - starting_image_count = len(self.dependent_var_int) - ending_image_count = starting_image_count + len(self.dependent_var_int) - - # last iteration Call - - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - k = {'SD': {'segment_model': self.model, 'sess': Mock()}} - - state = starting_image_count - new_state = self.subject.fit_transition( - state, self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), None, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, self.dummy_prev_weights, True, **k) - - image_count = new_state - self.assertEqual(ending_image_count, image_count) - - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session and sess.close must not get called for the middle buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - - # Non-last iteration Call - self.subject.K.set_session.reset_mock() - self.subject.K.clear_session.reset_mock() - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - k = {'SD': {'segment_model': self.model, 'sess': Mock()}} - - state = starting_image_count - new_state = self.subject.fit_transition( - state, self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), None, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, self.dummy_prev_weights, False, **k) - - image_count = new_state - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session and sess.close must not get called for the middle buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - - def test_fit_transition_multiple_model_middle_buffer_pass(self): - #TODO should we mock tensorflow's close_session and keras' - # clear_session instead of mocking the function `K.clear_session` - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - - starting_image_count = len(self.dependent_var_int) - ending_image_count = starting_image_count + len(self.dependent_var_int) - - # last iteration Call - - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - k = {'SD': {'segment_model': self.model, 'sess': Mock()}} - - state = starting_image_count - new_state = self.subject.fit_transition( - state, self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), None, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, self.dummy_prev_weights, True, True, **k) - - image_count = new_state - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session and sess.close must not get called for the middle buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - - def _test_fit_transition_last_buffer_pass(self, is_platform_pg): - #TODO should we mock tensorflow's close_session and keras' - # clear_session instead of mocking the function `K.clear_session` - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - self.subject.is_platform_pg = Mock(return_value = is_platform_pg) - - starting_image_count = 2*len(self.dependent_var_int) - ending_image_count = starting_image_count + len(self.dependent_var_int) - - # last iteration Call - - multiplied_weights = mult(self.total_images_per_seg[0],self.model_weights) - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - k = {'SD': {'segment_model' :self.model, 'sess': Mock()}} - - state = starting_image_count - new_state = self.subject.fit_transition( - state, self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), None, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, self.dummy_prev_weights, True, **k) - - state = np.fromstring(new_state, dtype=np.float32) - image_count = state[0] - weights = np.rint(state[1:]).astype(np.int) - self.assertEqual(ending_image_count, image_count) - # weights should be multiplied by final image count - self.assertTrue((multiplied_weights == weights).all()) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session and sess.close must get called for the last buffer in gpdb, - # but not in postgres - self.assertEqual(1, self.subject.K.clear_session.call_count) - - # Non-last iteration Call - self.subject.K.set_session.reset_mock() - self.subject.K.clear_session.reset_mock() - - multiplied_weights = mult(self.total_images_per_seg[0],self.model_weights) - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - k = {'SD': {'segment_model' :self.model, 'sess': Mock()}} - - state = starting_image_count - new_state = self.subject.fit_transition( - state, self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), None, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, self.dummy_prev_weights, False, **k) - - state = np.fromstring(new_state, dtype=np.float32) - image_count = state[0] - weights = np.rint(state[1:]).astype(np.int) - self.assertEqual(ending_image_count, image_count) - # weights should be multiplied by final image count - self.assertTrue((multiplied_weights == weights).all()) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session and sess.close must get called for the last buffer in gpdb, - # but not in postgres - self.assertEqual(0, self.subject.K.clear_session.call_count) - - def test_fit_transition_multiple_model_last_buffer_pass(self): - #TODO should we mock tensorflow's close_session and keras' - # clear_session instead of mocking the function `K.clear_session` - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - - starting_image_count = 2*len(self.dependent_var_int) - ending_image_count = starting_image_count + len(self.dependent_var_int) - - # last iteration Call - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - k = {'SD': {'segment_model' :self.model, 'sess': Mock()}} - - state = starting_image_count - new_state = self.subject.fit_transition( - state , self.dependent_var, self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), None, self.fit_params, 0, - self.dist_key_mapping, 0, 4, self.total_images_per_seg, False, - self.accessible_gpus_for_seg, self.dummy_prev_weights, True, True, **k) - - state = np.fromstring(new_state, dtype=np.float32) - weights = np.rint(state[0:]).astype(np.int) - - ## image count should not be added to the final state of - # fit multiple - self.assertEqual(len(self.model_weights), len(weights)) - - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # Clear session and sess.close must get called for the last buffer in gpdb, - # but not in postgres - self.assertEqual(1, self.subject.K.clear_session.call_count) - - def test_fit_transition_first_buffer_pass_pg(self): - self._test_fit_transition_first_buffer_pass(True) - - def test_fit_transition_first_buffer_pass_gpdb(self): - self._test_fit_transition_first_buffer_pass(False) - - def test_fit_transition_middle_buffer_pass_pg(self): - self._test_fit_transition_middle_buffer_pass(True) - - def test_fit_transition_middle_buffer_pass_gpdb(self): - self._test_fit_transition_middle_buffer_pass(False) - - def test_fit_transition_last_buffer_pass_pg(self): - self._test_fit_transition_last_buffer_pass(True) - - def test_fit_transition_last_buffer_pass_gpdb(self): - self._test_fit_transition_last_buffer_pass(False) - - def test_fit_transition_first_tuple_none_ind_var_dep_var(self): - k = {} - self.assertEqual('dummy_state', - self.subject.fit_transition('dummy_state', [0], None, - 'noshape', 'noshape', - 'dummy_model_json', "foo", "bar", - 1, [0,1,2], 0, 4, [3,3,3], False, - [0], 'dummy_prev_state', **k)) - self.assertEqual('dummy_state', - self.subject.fit_transition('dummy_state', None, [[0.5]], - 'noshape', 'noshape', - 'dummy_model_json', "foo", "bar", - 1, [0,1,2], 0, 4, [3,3,3], False, - [0], 'dummy_prev_state', **k)) - self.assertEqual('dummy_state', - self.subject.fit_transition('dummy_state', None, None, - 'noshape', 'noshape', - 'dummy_model_json', "foo", "bar", - 1, [0,1,2], 0, 4, [3,3,3], False, - [0], 'dummy_prev_state', **k)) - - def test_fit_merge(self): - image_count = self.total_images_per_seg[0] - state1 = [image_count] - state1.extend(mult(3,self.model_weights)) - state1 = np.array(state1, dtype=np.float32) - state2 = [image_count+30] - state2.extend(mult(2,self.model_weights)) - state2 = np.array(state2, dtype=np.float32) - merged_state = self.subject.fit_merge(state1.tostring(),state2.tostring()) - state = np.fromstring(merged_state, dtype=np.float32) - image_count_total = state[0] - weights = np.rint(state[1:]).astype(np.int) - - self.assertEqual( 2*image_count+30 , image_count_total ) - self.assertTrue( (mult(5,self.model_weights) == weights).all()) - - def test_fit_merge_none_first(self): - image_count = self.total_images_per_seg[0] - input_state = [image_count] - input_state.extend(self.model_weights) - input_state = np.array(input_state, dtype=np.float32) - merged_state = self.subject.fit_merge(None, input_state.tostring()) - state = np.fromstring(merged_state, dtype=np.float32) - image_count_total = state[0] - weights = np.rint(state[1:]).astype(np.int) - - self.assertEqual(image_count, image_count_total) - self.assertTrue((self.model_weights == weights).all()) - - def test_fit_merge_none_second(self): - image_count = self.total_images_per_seg[0] - input_state = [image_count] - input_state.extend(self.model_weights) - input_state = np.array(input_state, dtype=np.float32) - merged_state = self.subject.fit_merge(input_state.tostring(), None) - state = np.fromstring(merged_state, dtype=np.float32) - image_count_total = state[0] - weights = np.rint(state[1:]).astype(np.int) - - self.assertEqual(image_count, image_count_total) - self.assertTrue((self.model_weights == weights).all()) - - def test_fit_merge_both_none(self): - result = self.subject.fit_merge(None,None) - self.assertEqual(None, result) - - def test_fit_final(self): - image_count = self.total_images_per_seg[0] - input_state = [image_count] - input_state.extend(mult(image_count,self.model_weights)) - input_state = np.array(input_state, dtype=np.float32) - - output_state = self.subject.fit_final(input_state.tostring()) - output_state = np.fromstring(output_state, dtype=np.float32) - weights = np.rint(output_state).astype(np.int) - - self.assertTrue((self.model_weights == weights).all()) - - def test_fit_final_none(self): - result = self.subject.fit_final(None) - self.assertEqual(result, None) - - def test_fit_final_image_count_zero(self): - input_state = [0] - input_state.extend(self.model_weights) - input_state = np.array(input_state, dtype=np.float32) - - with self.assertRaises(plpy.PLPYException): - result = self.subject.fit_final(input_state.tostring()) - - def test_should_compute_metrics_this_iter_last_iter_case1(self): - res = self.subject.should_compute_metrics_this_iter(5, 5, 5) - self.assertEqual(True, res) - def test_should_compute_metrics_this_iter_last_iter_case2(self): - res = self.subject.should_compute_metrics_this_iter(5, 3, 5) - self.assertEqual(True, res) - def test_should_compute_metrics_this_iter_before_interval(self): - res = self.subject.should_compute_metrics_this_iter(1, 3, 5) - self.assertEqual(False, res) - def test_should_compute_metrics_this_iter_after_interval(self): - res = self.subject.should_compute_metrics_this_iter(4, 3, 5) - self.assertEqual(False, res) - def test_should_compute_metrics_this_iter_at_interval(self): - res = self.subject.should_compute_metrics_this_iter(3, 3, 5) - self.assertEqual(True, res) - def test_should_compute_metrics_this_iter_every_iter(self): - res = self.subject.should_compute_metrics_this_iter(2, 1, 5) - self.assertEqual(True, res) - - -class InternalKerasPredictTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import madlib_keras_predict - self.subject = madlib_keras_predict - - self.model = Sequential() - self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', - input_shape=(1,1,1,), padding='same')) - self.model.add(Flatten()) - - self.all_seg_ids = [0,1,2] - - self.independent_var = [[[240]]] - self.total_images_per_seg = [3,3,4] - self.subject.K.set_session = Mock() - self.subject.clear_keras_session = Mock() - - - def tearDown(self): - self.module_patcher.stop() - - def test_predict_first_image_pass_gpdb(self): - self.subject.is_platform_pg = Mock(return_value = False) - model_weights = [1, 2, 3, 4] - serialized_weights = np.array(model_weights, dtype=np.float32).tostring() - - k = {'SD': {}} - is_response = True - result = self.subject.internal_keras_predict( - self.independent_var, self.model.to_json(), - serialized_weights, is_response, 255, 0, self.all_seg_ids, - self.total_images_per_seg, False, 0, 4, **k) - self.assertEqual(1, len(result)) - self.assertEqual(1, k['SD']['row_count']) - self.assertEqual(True, 'segment_model_predict' in k['SD']) - - def test_predict_middle_image_pass_gpdb(self): - self.subject.is_platform_pg = Mock(return_value = False) - - k = {'SD': { 'row_count': 1}} - k['SD']['segment_model_predict'] = self.model - is_response = True - result = self.subject.internal_keras_predict( - self.independent_var, None, None, is_response, 255, 0, - self.all_seg_ids, self.total_images_per_seg, False, 0, 4, **k) - self.assertEqual(1, len(result)) - self.assertEqual(2, k['SD']['row_count']) - self.assertEqual(True, 'segment_model_predict' in k['SD']) - - def test_predict_last_image_pass_gpdb(self): - self.subject.is_platform_pg = Mock(return_value = False) - self.model.add(Dense(3)) - - k = {'SD': { 'row_count': 2}} - k['SD']['segment_model_predict'] = self.model - is_response = True - result = self.subject.internal_keras_predict( - self.independent_var, None, None, is_response, 255, 0, - self.all_seg_ids, self.total_images_per_seg, False, 0, 4, **k) - self.assertEqual(1, len(result)) - self.assertEqual(False, 'row_count' in k['SD']) - self.assertEqual(False, 'segment_model_predict' in k['SD']) - - k = {'SD': { 'row_count': 2}} - k['SD']['segment_model_predict'] = self.model - is_response = False - result = self.subject.internal_keras_predict( - self.independent_var, None, None, is_response, 255, 0, - self.all_seg_ids, self.total_images_per_seg, False, 0, 4, **k) - - # we except len(result) to be 3 because we have 3 dense layers in the - # architecture - self.assertEqual(3, len(result)) - self.assertEqual(False, 'row_count' in k['SD']) - self.assertEqual(False, 'segment_model_predict' in k['SD']) - - - def test_predict_error_should_clear_sd(self): - self.subject.is_platform_pg = Mock(return_value = False) - self.model.add(Dense(3)) - - # inject error by passing 0 as the normalizing const so that we get a - # divide by zero error - normalizing_const = 0 - k = {'SD':{}} - is_response = True - with self.assertRaises(plpy.PLPYException): - self.subject.internal_keras_predict( - self.independent_var, None, None, is_response, normalizing_const, - 0, self.all_seg_ids, self.total_images_per_seg, False, 0, 4, **k) - self.assertEqual(False, 'row_count' in k['SD']) - self.assertEqual(False, 'segment_model_predict' in k['SD']) - - -class MadlibKerasPredictBYOMTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - self.num_classes = 5 - self.model = Sequential() - self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', - input_shape=(1,1,1,), padding='same')) - self.model.add(Dense(self.num_classes)) - - self.pred_type = 'prob' - self.use_gpus = False - self.class_values = ['foo', 'bar', 'baaz', 'foo2', 'bar2'] - self.normalizing_const = 255.0 - - import madlib_keras_predict - self.module = madlib_keras_predict - self.module.get_model_arch_weights = Mock(return_value=( - self.model.to_json(), 'weights')) - self.module.InputValidator.validate_predict_byom_tables = Mock() - self.module.InputValidator.validate_input_shape = Mock() - self.module.BasePredict.call_internal_keras = Mock() - - def tearDown(self): - self.module_patcher.stop() - - def test_predictbyom_defaults_1(self): - self.module.get_accessible_gpus_for_seg = Mock(return_value = [2,2,2]) - res = self.module.PredictBYOM('schema_madlib', 'model_arch_table', - 'model_id', 'test_table', 'id_col', - 'independent_varname', 'output_table', None, - True, None, None) - self.assertEqual('response', res.pred_type) - self.assertEqual(2, res.gpus_per_host) - self.assertEqual([0,1,2,3,4], res.class_values) - self.assertEqual(1.0, res.normalizing_const) - self.assertEqual('text', res.dependent_vartype) - - def test_predictbyom_defaults_2(self): - res = self.module.PredictBYOM('schema_madlib', 'model_arch_table', - 'model_id', 'test_table', 'id_col', - 'independent_varname', 'output_table', - self.pred_type, self.use_gpus, - self.class_values, self.normalizing_const) - self.assertEqual('prob', res.pred_type) - self.assertEqual(0, res.gpus_per_host) - self.assertEqual(['foo', 'bar', 'baaz', 'foo2', 'bar2'], res.class_values) - self.assertEqual(255.0, res.normalizing_const) - self.assertEqual('double precision', res.dependent_vartype) - - def test_predictbyom_exception_invalid_params(self): - with self.assertRaises(plpy.PLPYException) as error: - self.module.PredictBYOM('schema_madlib', 'model_arch_table', - 'model_id', 'test_table', 'id_col', - 'independent_varname', 'output_table', - 'invalid_pred_type', self.use_gpus, - self.class_values, self.normalizing_const) - self.assertIn('invalid_pred_type', str(error.exception)) - - with self.assertRaises(plpy.PLPYException) as error: - self.module.PredictBYOM('schema_madlib', 'model_arch_table', - 'model_id', 'test_table', 'id_col', - 'independent_varname', 'output_table', - self.pred_type, self.use_gpus, - ["foo", "bar", "baaz"], self.normalizing_const) - self.assertIn('class values', str(error.exception).lower()) - - with self.assertRaises(plpy.PLPYException) as error: - self.module.PredictBYOM('schema_madlib', 'model_arch_table', - 'model_id', 'test_table', 'id_col', - 'independent_varname', 'output_table', - self.pred_type, self.use_gpus, - self.class_values, 0) - self.assertIn('normalizing const', str(error.exception).lower()) - - -class MadlibKerasWrapperTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - - import madlib_keras_wrapper - self.subject = madlib_keras_wrapper - - self.use_gpus = False - - def tearDown(self): - self.module_patcher.stop() - - def test_get_gpu_memory_fraction(self): - - gpus_per_host = 4 - segments_per_host = 4 - result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host) - self.assertEqual(result, 0.9) - - gpus_per_host = 10 - segments_per_host = 4 - result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host) - self.assertEqual(result, 0.9) - - gpus_per_host = 2 - segments_per_host = 6 - result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host) - self.assertEqual(result, 0.3) - - gpus_per_host = 1 - segments_per_host = 4 - result = self.subject.get_gpu_memory_fraction(gpus_per_host, segments_per_host) - self.assertEqual(result, 0.225) - - def test_get_device_name_and_set_cuda_env_postgres(self): - self.subject.is_platform_pg = Mock(return_value = True) - - seg_id = -1 - gpus_per_host = 3 - - self.assertEqual('/gpu:0', self.subject.get_device_name_and_set_cuda_env( - gpus_per_host, seg_id )) - self.assertEqual('0,1,2', os.environ['CUDA_VISIBLE_DEVICES']) - - gpus_per_host = 0 - self.assertEqual('/cpu:0', self.subject.get_device_name_and_set_cuda_env( - gpus_per_host, seg_id )) - self.assertEqual('-1', os.environ['CUDA_VISIBLE_DEVICES']) - - def test_get_device_name_and_set_cuda_env_gpdb(self): - self.subject.is_platform_pg = Mock(return_value = False) - - seg_id=3 - gpus_per_host=2 - self.assertEqual('/gpu:0', self.subject.get_device_name_and_set_cuda_env( - gpus_per_host, seg_id)) - self.assertEqual('1', os.environ['CUDA_VISIBLE_DEVICES']) - - gpus_per_host=0 - self.assertEqual('/cpu:0', self.subject.get_device_name_and_set_cuda_env( - gpus_per_host, seg_id)) - self.assertEqual('-1', os.environ['CUDA_VISIBLE_DEVICES']) - - - def test_split_and_strip(self): - self.assertEqual(('a','b'), self.subject.split_and_strip(' a = b ')) - - def test_parse_optimizer(self): - opt_name = 'SGD' - final_args = {'lr':0.01, 'decay':1e-6, 'nesterov':True} - compile_dict = {} - compile_dict['optimizer']='SGD(lr=0.01, decay=1e-6, nesterov=True)' - result_name, result_params = self.subject.parse_optimizer(compile_dict) - - self.assertEqual(result_name, opt_name) - self.assertDictEqual(result_params, final_args) - - def test_validate_and_literal_eval_keys(self): - test_dict = {'batch_size':'2', 'epochs':'1', 'verbose':'0'} - target_dict = {'batch_size':2, 'epochs':1, 'verbose':0} - literal_eval_fit_params = ['batch_size','epochs','verbose','shuffle', - 'class_weight','initial_epoch','steps_per_epoch'] - accepted_fit_params = literal_eval_fit_params + ['shuffle'] - result_params = self.subject.validate_and_literal_eval_keys( - test_dict, - literal_eval_fit_params, - accepted_fit_params) - self.assertDictEqual(result_params, target_dict) - - def test_parse_and_validate_fit_params(self): - result = {'batch_size':2, 'epochs':1, 'verbose':0} - self.assertDictEqual(result, self.subject.parse_and_validate_fit_params('batch_size=2, epochs=1, verbose=0')) - - def test_parse_and_validate_fit_params(self): - test_str = "batch_size=2, epochs=1, verbose=0" - fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0} - result_params = self.subject.parse_and_validate_fit_params(test_str) - self.assertDictEqual(result_params, fit_dict) - - test_str = "batch_size=2, epochs=1, verbose=0, shuffle = 'batch'" - fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0, 'shuffle':'batch'} - result_params = self.subject.parse_and_validate_fit_params(test_str) - self.assertDictEqual(result_params, fit_dict) - - test_str = "batch_size=2, epochs=1, verbose=0, shuffle = True" - fit_dict = {'batch_size':2, 'epochs':1, 'verbose':0, 'shuffle':True} - result_params = self.subject.parse_and_validate_fit_params(test_str) - self.assertDictEqual(result_params, fit_dict) - - test_str = "" - fit_dict = {} - result_params = self.subject.parse_and_validate_fit_params(test_str) - self.assertDictEqual(result_params, fit_dict) - - def test_parse_and_validate_fit_params_duplicate_params(self): - test_str = "batch_size=1, batch_size=10, verbose=0" - fit_dict = {'batch_size':10, 'verbose':0} - result_params = self.subject.parse_and_validate_fit_params(test_str) - self.assertDictEqual(result_params, fit_dict) - - def test_parse_and_validate_fit_params_invalid_fit_param_fail(self): - test_str = "does_not_exist=2, epochs=1, verbose=0" - with self.assertRaises(plpy.PLPYException) as error: - self.subject.parse_and_validate_fit_params(test_str) - - self.assertIn('not currently accepted', str(error.exception)) - - test_str = "batch_size=not_lit_eval(1), epochs=1, verbose=0" - with self.assertRaises(plpy.PLPYException) as error: - self.subject.parse_and_validate_fit_params(test_str) - - self.assertIn('invalid input value', str(error.exception)) - - def test_parse_and_validate_compile_params(self): - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ - "loss='categorical_crossentropy', metrics=['accuracy']" - compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', - 'metrics':['accuracy'], 'loss':'categorical_crossentropy'} - opt_name,opt_args,result_params = self.subject.parse_and_validate_compile_params(test_str) - self.assertDictEqual(result_params, compile_dict) - - # Test without the metrics - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ - "loss='categorical_crossentropy'" - compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', - 'loss':'categorical_crossentropy'} - opt_name,opt_args,result_params = self.subject.parse_and_validate_compile_params(test_str) - self.assertDictEqual(result_params, compile_dict) - - def test_validate_metrics_None_pass(self): - compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', - 'metrics':['accuracy'], 'loss':'categorical_crossentropy'} - self.subject._validate_metrics(compile_dict) - - def test_validate_metrics_empty_pass(self): - compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', - 'metrics':[], 'loss':'categorical_crossentropy'} - self.subject._validate_metrics(compile_dict) - - def test_validate_metrics_two_params_fail(self): - compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', - 'metrics':['accuracy','mae'], 'loss':'categorical_crossentropy'} - with self.assertRaises(plpy.PLPYException) as error: - self.subject._validate_metrics(compile_dict) - - def test_validate_metrics_one_unsupported_fail(self): - compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', - 'metrics':['sparse_categorical_accuracy'], 'loss':'categorical_crossentropy'} - with self.assertRaises(plpy.PLPYException) as error: - self.subject._validate_metrics(compile_dict) - - def test_validate_metrics_two_unsupported_fail(self): - compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', - 'metrics':['sparse_categorical_accuracy', 'sparse_categorical_crossentropy'], 'loss':'categorical_crossentropy'} - with self.assertRaises(plpy.PLPYException) as error: - self.subject._validate_metrics(compile_dict) - - def test_validate_metrics_one_supported_one_unsupported_fail(self): - compile_dict = {'optimizer':'SGD(lr=0.01, decay=1e-6, nesterov=True)', - 'metrics':['accuracy', 'sparse_categorical_crossentropy'], 'loss':'categorical_crossentropy'} - with self.assertRaises(plpy.PLPYException) as error: - self.subject._validate_metrics(compile_dict) - - def test_parse_and_validate_compile_params_default_optimizer_pass(self): - test_str = "optimizer='SGD', loss='categorical_crossentropy'" - _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str) - self.assertEqual('SGD', result_dict['optimizer']) - - test_str = "optimizer='sgd()', loss='categorical_crossentropy'" - _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str) - self.assertEqual('sgd()', result_dict['optimizer']) - - def test_parse_and_validate_compile_params_duplicate_loss(self): - test_str = "optimizer='SGD', loss='loss1', metrics=['accuracy'], loss='loss2'" - _,_,result_dict = self.subject.parse_and_validate_compile_params(test_str) - self.assertEqual('loss2', result_dict['loss']) - - def test_parse_and_validate_compile_params_no_optimizer_fail(self): - test_str = "loss='categorical_crossentropy', metrics=['accuracy']" - - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_compile_params(test_str) - - def test_parse_and_validate_compile_params_no_loss_fail(self): - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ - "metrics=['accuracy']" - - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_compile_params(test_str) - - def test_parse_and_validate_compile_params_dict_metrics_fail(self): - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), " \ - "loss='categorical_crossentropy', metrics={'0':'accuracy'}" - - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_compile_params(test_str) - - def test_parse_and_validate_compile_params_tensor_loss_weights_fail(self): - - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \ - " loss='categorical_crossentropy', metrics=['accuracy']," \ - " loss_weights = keras.layers.Input(shape=(32,))" - - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_compile_params(test_str) - - def test_parse_and_validate_compile_params_list_dict_sample_weight_mode_fail(self): - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \ - " loss='categorical_crossentropy', metrics=['accuracy']," \ - " sample_weight_mode = [0,1]" - - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_compile_params(test_str) - - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \ - " loss='categorical_crossentropy', metrics=['accuracy']," \ - " sample_weight_mode = {'0':'1'}" - - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_compile_params(test_str) - - - def test_parse_and_validate_compile_params_target_tensors_fail(self): - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True)," \ - " loss='categorical_crossentropy', metrics=['accuracy']," \ - " target_tensors = keras.layers.Input(shape=(32,))" - - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_compile_params(test_str) - - def test_parse_and_validate_fit_params_callbacks_fail(self): - - test_str = "batch_size=2, epochs=1, verbose=0, callbacks=keras.callbacks.Callback()" - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_fit_params(test_str) - - def test_parse_and_validate_fit_params_validation_split_fail(self): - - test_str = "batch_size=2, epochs=1, verbose=0, validation_split=0.1" - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_fit_params(test_str) - - def test_parse_and_validate_fit_params_validation_data_fail(self): - - test_str = "batch_size=2, epochs=1, verbose=0, validation_data=(1,2,3)" - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_fit_params(test_str) - - def test_parse_and_validate_fit_params_sample_weight_fail(self): - - test_str = "batch_size=2, epochs=1, verbose=0, sample_weight=np.array([1,2,3])" - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_fit_params(test_str) - - def test_parse_and_validate_fit_params_validation_steps_fail(self): - - test_str = "batch_size=2, epochs=1, verbose=0, validation_steps=1" - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_fit_params(test_str) - - def test_parse_and_validate_fit_params_validation_freq_fail(self): - test_str = "batch_size=2, epochs=1, verbose=0, validation_freq=1" - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_fit_params(test_str) - - test_str = "batch_size=2, epochs=1, verbose=0, validation_freq=[1]" - with self.assertRaises(plpy.PLPYException): - self.subject.parse_and_validate_fit_params(test_str) - - def test_parse_and_validate_compile_params_syntax_error_fail(self): - #missing end parentheses - test_str = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True" - with self.assertRaises(ValueError) as error: - self.subject.parse_and_validate_compile_params(test_str) - self.assertIn('could not convert string to float', str(error.exception)) - - #missing beginning parentheses - test_str = "optimizer=SGDlr=0.01, decay=1e-6, nesterov=True)," \ - " loss='categorical_crossentropy'" - with self.assertRaises(plpy.PLPYException) as error: - self.subject.parse_and_validate_compile_params(test_str) - self.assertIn('not currently accepted', str(error.exception)) - - #missing comma - test_str = "optimizer=SGD(lr=0.01 decay=1e-6, nesterov=True)," \ - " loss='categorical_crossentropy'" - with self.assertRaises(ValueError) as error: - self.subject.parse_and_validate_compile_params(test_str) - self.assertIn('invalid literal for float', str(error.exception)) - - def test_parse_and_validate_fit_params_invalid_optimizer_fail(self): - test_str = "optimizer='SGD1', loss='categorical_crossentropy'" - with self.assertRaises(plpy.PLPYException) as error: - self.subject.parse_and_validate_compile_params(test_str) - self.assertIn('invalid optimizer', str(error.exception)) - - -class MadlibKerasFitCommonValidatorTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import madlib_keras_validator - self.subject = madlib_keras_validator - - def tearDown(self): - self.module_patcher.stop() - - - def test_is_valid_metrics_compute_frequency_True_None(self): - self.subject.FitCommonValidator._validate_common_args = Mock() - obj = self.subject.FitCommonValidator( - 'test_table', 'val_table', 'model_table', 'model_arch_table', 2, - 'dep_varname', 'independent_varname', 5, None, False, False, [0], - 'module_name') - self.assertEqual(True, obj._is_valid_metrics_compute_frequency()) - - def test_is_valid_metrics_compute_frequency_True_num(self): - self.subject.FitCommonValidator._validate_common_args = Mock() - obj = self.subject.FitCommonValidator( - 'test_table', 'val_table', 'model_table', 'model_arch_table', 2, - 'dep_varname', 'independent_varname', 5, 3, False, False, [0], - 'module_name') - self.assertEqual(True, obj._is_valid_metrics_compute_frequency()) - - def test_is_valid_metrics_compute_frequency_False_zero(self): - self.subject.FitCommonValidator._validate_common_args = Mock() - obj = self.subject.FitCommonValidator( - 'test_table', 'val_table', 'model_table', 'model_arch_table', 2, - 'dep_varname', 'independent_varname', 5, 0, False, False, [0], - 'module_name') - self.assertEqual(False, obj._is_valid_metrics_compute_frequency()) - - def test_is_valid_metrics_compute_frequency_False_greater(self): - self.subject.FitCommonValidator._validate_common_args = Mock() - obj = self.subject.FitCommonValidator( - 'test_table', 'val_table', 'model_table', 'model_arch_table', 2, - 'dep_varname', 'independent_varname', 5, 6, False, False, [0], - 'module_name') - self.assertEqual(False, obj._is_valid_metrics_compute_frequency()) - - -class InputValidatorTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import madlib_keras_validator - self.module = madlib_keras_validator - self.subject = self.module.InputValidator - - self.module_name = 'module' - self.test_table = 'test_table' - self.model_table = 'model_table' - self.id_col = 'id_col' - self.ind_var = 'ind_var' - self.model_arch_table = 'model_arch_table' - self.model_id = 2 - self.num_classes = 1598 - self.model = Sequential() - self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', - input_shape=(1,1,1,), padding='same')) - self.model.add(Dense(self.num_classes)) - self.classes = ['train', 'boat', 'car', 'airplane'] - - def tearDown(self): - self.module_patcher.stop() - - def test_validate_pred_type_invalid_pred_type(self): - with self.assertRaises(plpy.PLPYException) as error: - self.subject.validate_pred_type( - self.module_name, 'invalid_pred_type', ['cat', 'dog']) - self.assertIn('type', str(error.exception).lower()) - - def test_validate_class_values_greater_than_1600_class_values(self): - self.model.add(Dense(1599)) - with self.assertRaises(plpy.PLPYException) as error: - self.subject.validate_class_values( - self.module_name, range(1599), 'prob', self.model.to_json()) - self.assertIn('1600', str(error.exception)) - - def test_validate_class_values_valid_class_values_prob(self): - self.subject.validate_class_values( - self.module_name, range(self.num_classes), 'prob', self.model.to_json()) - self.subject.validate_class_values( - self.module_name, None, 'prob', self.model.to_json()) - - def test_validate_class_values_valid_pred_type_valid_class_values_response(self): - self.subject.validate_class_values( - self.module_name, range(self.num_classes), 'response', self.model.to_json()) - self.subject.validate_class_values( - self.module_name, None, 'response', self.model.to_json()) - - def test_validate_input_shape_shapes_do_not_match(self): - # minibatched data - self.plpy_mock_execute.return_value = [{'shape': [1,3,32,32]}] - with self.assertRaises(plpy.PLPYException): - self.subject.validate_input_shape( - self.test_table, self.ind_var, [32,32,3], 2, True) - # non-minibatched data - self.plpy_mock_execute.return_value = [{'n_0': 1,'n_1': 32,'n_2': 32,'n_3': 3}] - with self.assertRaises(plpy.PLPYException): - self.subject.validate_input_shape( - self.test_table, self.ind_var, [32,32,3], 1) - self.plpy_mock_execute.return_value = [{'n_0': 1,'n_1': 3}] - with self.assertRaises(plpy.PLPYException): - self.subject.validate_input_shape( - self.test_table, self.ind_var, [3,32], 1) - - def test_validate_input_shape_shapes_match(self): - # minibatched data - self.plpy_mock_execute.return_value = [{'shape': [1,32,32,3]}] - self.subject.validate_input_shape( - self.test_table, self.ind_var, [32,32,3], 2, True) - # non-minibatched data - self.plpy_mock_execute.return_value = [{'n_0': 32,'n_1': 32,'n_2': 3}] - self.subject.validate_input_shape( - self.test_table, self.ind_var, [32,32,3], 1) - - def test_validate_model_arch_table_none_values(self): - with self.assertRaises(plpy.PLPYException) as error: - obj = self.subject.validate_model_arch_table( - self.module_name, None, self.model_id) - self.assertIn('null', str(error.exception).lower()) - - self.module.input_tbl_valid = Mock() - with self.assertRaises(plpy.PLPYException) as error: - obj = self.subject.validate_model_arch_table( - self.module_name, self.model_arch_table, None) - self.assertIn('id', str(error.exception).lower()) - - def test_validate_gpu_config_with_gpu_all_segments(self): - - self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': 'all_segments'}] - obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1]) - - def test_validate_gpu_config_no_gpu_all_segments(self): - - self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': 'all_segments'}] - with self.assertRaises(plpy.PLPYException) as error: - obj = self.subject._validate_gpu_config(self.module_name, 'foo', [0]) - self.assertIn('missing gpus', str(error.exception).lower()) - - def test_validate_gpu_config_with_gpu_valid_seg_list(self): - - self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': [0,1]}] - obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1,1,0,1]) - - def test_validate_gpu_config_with_gpu_invalid_seg_list(self): - - self.plpy_mock_execute.return_value = [{'__internal_gpu_config__': [0,1]}] - with self.assertRaises(plpy.PLPYException) as error: - obj = self.subject._validate_gpu_config(self.module_name, 'foo', [1,0,0,1]) - self.assertIn('does not have gpu', str(error.exception).lower()) - -class MadlibSerializerTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import madlib_keras_serializer - self.subject = madlib_keras_serializer - - def tearDown(self): - self.module_patcher.stop() - - def test_deserialize_image_1d_weights_null_state_returns_none(self): - self.assertEqual(None, self.subject.deserialize_as_image_1d_weights(None)) - - def test_deserialize_image_1d_weights_returns_not_none(self): - dummy_state = np.array([2,3,4,5,6], dtype=np.float32) - res = self.subject.deserialize_as_image_1d_weights(dummy_state.tostring()) - self.assertEqual(2, res[0]) - self.assertEqual([3,4,5,6], res[1].tolist()) - - def test_deserialize_nd_weights_null_input_returns_none(self): - dummy_state = np.array([0,1,2,3,4,5,6], dtype=np.float32) - self.assertEqual(None, self.subject.deserialize_as_nd_weights(dummy_state.tostring(), None)) - self.assertEqual(None, self.subject.deserialize_as_nd_weights(None, [1, 2, 3])) - self.assertEqual(None, self.subject.deserialize_as_nd_weights(None, None)) - - def test_deserialize_nd_weights_valid_input_returns_not_none(self): - dummy_model_weights = np.array([3,4,5], dtype=np.float32) - dummy_model_shape = [(2, 1, 1, 1), (1,)] - res = self.subject.deserialize_as_nd_weights(dummy_model_weights.tostring(), - dummy_model_shape) - self.assertEqual([[[[3.0]]], [[[4.0]]]], res[0].tolist()) - self.assertEqual([5], res[1].tolist()) - - def test_deserialize_nd_weights_invalid_input_fails(self): - # pass an invalid state with missing model weights - invalid_model_weights = np.array([1,2], dtype=np.float32) - dummy_model_shape = [(2, 1, 1, 1), (1,)] - - # we expect raise exception because we cannot reshape - # model weights of size 0 into shape (2,2,3,1) - with self.assertRaises(plpy.PLPYException) as error: - self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(), - dummy_model_shape) - - invalid_model_weights = np.array([1,2,3,4], dtype=np.float32) - dummy_model_shape = [(2, 2, 3, 1), (1,)] - # we expect raise exception because we cannot reshape - # model weights of size 2 into shape (2,2,3,1) - with self.assertRaises(plpy.PLPYException) as error: - self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(), - dummy_model_shape) - - invalid_model_weights = np.array([0,1,2,3,4], dtype=np.float32) - dummy_model_shape = [(2, 1), (1,)] - # we expect to raise exception because we cannot reshape - # model weights of size 2 into shape (1,) - with self.assertRaises(plpy.PLPYException) as error: - self.subject.deserialize_as_nd_weights(invalid_model_weights.tostring(), - dummy_model_shape) - - def test_serialize_image_nd_weights_none_weights_returns_none(self): - res = self.subject.serialize_state_with_nd_weights(0, None) - self.assertEqual(None , res) - - def test_serialize_image_nd_weights_valid_output(self): - res = self.subject.serialize_state_with_nd_weights(0, [np.array([1, 3]), - np.array([4,5])]) - self.assertEqual(np.array([0,1,3,4,5], dtype=np.float32).tostring(), - res) - - def test_serialize_image_1d_weights_none_weights_returns_none(self): - res = self.subject.serialize_state_with_1d_weights(0, None) - self.assertEqual(None , res) - - def test_serialize_image_1d_weights_valid_output(self): - res = self.subject.serialize_state_with_1d_weights(0, np.array([1, 3, 4, 5])) - self.assertEqual(np.array([0,1,3,4,5], dtype=np.float32).tostring(), - res) - - -class MadlibKerasHelperTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import madlib_keras_helper - self.subject = madlib_keras_helper - self.input_data = [32, 32, 3] - - def tearDown(self): - self.module_patcher.stop() - - def test_expand_input_dims(self): - self.assertEqual(np.array(self.input_data).shape, (3,)) - res = self.subject.expand_input_dims(self.input_data) - self.assertEqual(res.shape, (1, 3)) - - def test_strip_trailing_nulls_from_class_values(self): - self.assertEqual(['cat', 'dog'], - self.subject.strip_trailing_nulls_from_class_values( - ['cat', 'dog'])) - self.assertEqual([None, 'cat', 'dog'], - self.subject.strip_trailing_nulls_from_class_values( - [None, 'cat', 'dog'])) - self.assertEqual([None, 'cat', 'dog'], - self.subject.strip_trailing_nulls_from_class_values( - [None, 'cat', 'dog', None, None])) - self.assertEqual(['cat', 'dog'], - self.subject.strip_trailing_nulls_from_class_values( - ['cat', 'dog', None, None])) - self.assertEqual([None], - self.subject.strip_trailing_nulls_from_class_values( - [None, None])) - - def test_get_gpus_per_one_seg_gpu_gpdb(self): - - self.subject.is_platform_pg = Mock(return_value = False) - - self.plpy_mock_execute.side_effect = \ - [ [], - [ {'hostname': 'mdw0', 'count' : 1}], - [], - [ {'hostname': 'mdw0', 'segment_id' : 0}, - {'hostname': 'mdw1', 'segment_id' : 1}, - {'hostname': 'mdw2', 'segment_id' : 2} - ]] - - self.assertEqual([1,0,0], self.subject.get_accessible_gpus_for_seg( - 'schema_madlib', 2, 'foo')) - - def test_get_gpus_per_mult_seg_gpu_gpdb(self): - - self.subject.is_platform_pg = Mock(return_value = False) - - self.plpy_mock_execute.side_effect = \ - [[], - [ {'hostname': 'mdw0', 'count' : 1}], - [], - [ {'hostname': 'mdw0', 'segment_id' : 0}, - {'hostname': 'mdw0', 'segment_id' : 1}, - {'hostname': 'mdw1', 'segment_id' : 2}, - {'hostname': 'mdw1', 'segment_id' : 3} - ]] - - self.assertEqual([1,1,0,0], self.subject.get_accessible_gpus_for_seg( - 'schema_madlib', 2, 'foo')) - - def test_get_gpus_per_no_gpu_gpdb(self): - - self.subject.is_platform_pg = Mock(return_value = False) - - self.plpy_mock_execute.side_effect = [[],[],[]] - with self.assertRaises(plpy.PLPYException) as error: - self.subject.get_accessible_gpus_for_seg('schema_madlib', 2, 'foo') - self.assertIn('no gpus configured on hosts', str(error.exception).lower()) - -class MadlibKerasEvaluationTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import madlib_keras - self.subject = madlib_keras - - self.model = Sequential() - self.model.add(Conv2D(2, kernel_size=(1, 1), activation='relu', - input_shape=(1,1,1,), padding='same')) - self.model.add(Flatten()) - - self.compile_params = "optimizer=SGD(lr=0.01, decay=1e-6, nesterov=True), loss='categorical_crossentropy', metrics=['accuracy']" - self.model_weights = [3,4,5,6] - self.serialized_weights = np.array(self.model_weights, dtype='float32' - ).tostring() - self.loss = 0.5947071313858032 - self.accuracy = 1.0 - self.dist_key_mapping = [0,1,2] - self.accessible_gpus_for_seg = [0] - - #self.model.evaluate = Mock(return_value = [self.loss, self.accuracy]) - - self.independent_var_real = [[[[0.5]]]] * 10 - self.dependent_var_int = [[0,1]] * 10 - - # Params as bytea - self.independent_var = np.array(self.independent_var_real, dtype=np.float32).tobytes() - self.dependent_var = np.array(self.dependent_var_int, dtype=np.int16).tobytes() - - self.independent_var_shape = [10,1,1,1] - self.dependent_var_shape = [10,2] - # We test on segment 0, which has 3 buffers filled with 10 identical - # images each, or 30 images total - self.total_images_per_seg = [3*len(self.dependent_var_int),20,40] - - def tearDown(self): - self.module_patcher.stop() - - def _test_internal_keras_eval_transition_first_buffer(self, is_platform_pg): - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - self.subject.is_platform_pg = Mock(return_value = is_platform_pg) - starting_image_count = 0 - ending_image_count = len(self.dependent_var_int) - - # last iteration call - - k = {'SD' : {}} - state = [0,0,0] - new_state = self.subject.internal_keras_eval_transition( - state, self.dependent_var , self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), - self.serialized_weights, self.compile_params, 0, - self.dist_key_mapping, 0, 4, - self.total_images_per_seg, False, self.accessible_gpus_for_seg, True, **k) - - agg_loss, agg_accuracy, image_count = new_state - - self.assertEqual(ending_image_count, image_count) - # Call set_session once for gpdb (but not for postgres) - self.assertEqual(1, self.subject.K.set_session.call_count) - # loss and accuracy should be unchanged - self.assertAlmostEqual(self.loss * image_count, agg_loss, 4) - self.assertAlmostEqual(self.accuracy * image_count, agg_accuracy, 4) - # Clear session and sess.close must not get called for the first buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - self.assertTrue(k['SD']['segment_model']) - - # Non-final call - - self.subject.K.set_session.reset_mock() - self.subject.K.clear_session.reset_mock() - k = {'SD' : {}} - state = [0,0,0] - - new_state = self.subject.internal_keras_eval_transition( - state, self.dependent_var , self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), - self.serialized_weights, self.compile_params, 0, - self.dist_key_mapping, 0, 4, - self.total_images_per_seg, False, self.accessible_gpus_for_seg, False, **k) - agg_loss, agg_accuracy, image_count = new_state - - self.assertEqual(ending_image_count, image_count) - # set_session must not get called for the first buffer - self.assertEqual(1, self.subject.K.set_session.call_count) - # loss and accuracy should be unchanged - self.assertAlmostEqual(self.loss * image_count, agg_loss, 4) - self.assertAlmostEqual(self.accuracy * image_count, agg_accuracy, 4) - # Clear session and sess.close must not get called for the first buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - self.assertTrue(k['SD']['segment_model']) - - def _test_internal_keras_eval_transition_middle_buffer(self, is_platform_pg): - #TODO should we mock tensorflow's close_session and keras' - # clear_session instead of mocking the function `K.clear_session` - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - self.subject.is_platform_pg = Mock(return_value = is_platform_pg) - - starting_image_count = len(self.dependent_var_int) - ending_image_count = starting_image_count + len(self.dependent_var_int) - - # last iteration call - - k = {'SD' : {}} - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - - state = [self.loss * starting_image_count, self.accuracy * starting_image_count, starting_image_count] - k['SD']['segment_model'] = self.model - k['SD']['sess'] = Mock() - - new_state = self.subject.internal_keras_eval_transition( - state, self.dependent_var , self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), - 'dummy_model_weights', None, 0, - self.dist_key_mapping, 0, 4, - self.total_images_per_seg, False, self.accessible_gpus_for_seg, True, **k) - - agg_loss, agg_accuracy, image_count = new_state - - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # loss and accuracy should be unchanged - self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4) - self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4) - # Clear session and sess.close must not get called for the middle buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - - # Non-last iteration call - - self.subject.K.set_session.reset_mock() - self.subject.K.clear_session.reset_mock() - k = {'SD' : {}} - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - - state = [self.loss * starting_image_count, self.accuracy * starting_image_count, starting_image_count] - k['SD']['segment_model'] = self.model - k['SD']['sess'] = Mock() - - new_state = self.subject.internal_keras_eval_transition( - state, self.dependent_var , self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), - 'dummy_model_weights', None, 0, - self.dist_key_mapping, 0, 4, - self.total_images_per_seg, False, self.accessible_gpus_for_seg, False, **k) - agg_loss, agg_accuracy, image_count = new_state - - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # loss and accuracy should be unchanged - self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4) - self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4) - # Clear session and sess.close must not get called for the middle buffer - self.assertEqual(0, self.subject.K.clear_session.call_count) - - def _test_internal_keras_eval_transition_last_buffer(self, is_platform_pg): - #TODO should we mock tensorflow's close_session and keras' - # clear_session instead of mocking the function `K.clear_session` - self.subject.K.set_session = Mock() - self.subject.K.clear_session = Mock() - self.subject.is_platform_pg = Mock(return_value = is_platform_pg) - - starting_image_count = 2*len(self.dependent_var_int) - ending_image_count = starting_image_count + len(self.dependent_var_int) - - k = {'SD' : {}} - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - - state = [self.loss * starting_image_count, - self.accuracy * starting_image_count, - starting_image_count] - - k['SD']['segment_model'] = self.model - k['SD']['sess'] = Mock() - - new_state = self.subject.internal_keras_eval_transition( - state, self.dependent_var , self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), - 'dummy_model_weights', None, 0, - self.dist_key_mapping, 0, 4, - self.total_images_per_seg, False, self.accessible_gpus_for_seg, True, **k) - agg_loss, agg_accuracy, image_count = new_state - - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # loss and accuracy should be unchanged - self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4) - self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4) - # Clear session and sess.close must get called for the last buffer in gpdb, - # but not in postgres - self.assertEqual(1, self.subject.K.clear_session.call_count) - - # Non-final call - - self.subject.K.set_session.reset_mock() - self.subject.K.clear_session.reset_mock() - k = {'SD' : {}} - - self.subject.compile_and_set_weights(self.model, self.compile_params, - '/cpu:0', self.serialized_weights) - - state = [self.loss * starting_image_count, - self.accuracy * starting_image_count, - starting_image_count] - - k['SD']['segment_model'] = self.model - k['SD']['sess'] = Mock() - - new_state = self.subject.internal_keras_eval_transition( - state, self.dependent_var , self.independent_var, - self.dependent_var_shape, self.independent_var_shape, - self.model.to_json(), - 'dummy_model_weights', None, 0, - self.dist_key_mapping, 0, 4, - self.total_images_per_seg, False, self.accessible_gpus_for_seg, False, **k) - - agg_loss, agg_accuracy, image_count = new_state - - self.assertEqual(ending_image_count, image_count) - # set_session is always called - self.assertEqual(1, self.subject.K.set_session.call_count) - # loss and accuracy should be unchanged - self.assertAlmostEqual(self.loss * ending_image_count, agg_loss, 4) - self.assertAlmostEqual(self.accuracy * ending_image_count, agg_accuracy, 4) - # Clear session and sess.close must not get called in non-final iterations - self.assertEqual(0, self.subject.K.clear_session.call_count) - - def test_internal_keras_eval_transition_first_buffer_pg(self): - self._test_internal_keras_eval_transition_first_buffer(True) - - def test_internal_keras_eval_transition_first_buffer_gpdb(self): - self._test_internal_keras_eval_transition_first_buffer(False) - - def test_internal_keras_eval_transition_middle_buffer_pg(self): - self._test_internal_keras_eval_transition_middle_buffer(True) - - def test_internal_keras_eval_transition_middle_buffer_gpdb(self): - self._test_internal_keras_eval_transition_middle_buffer(False) - - def test_internal_keras_eval_transition_last_buffer_pg(self): - self._test_internal_keras_eval_transition_last_buffer(True) - - def test_internal_keras_eval_transition_last_buffer_gpdb(self): - self._test_internal_keras_eval_transition_last_buffer(False) - - def test_internal_keras_eval_merge(self): - image_count = self.total_images_per_seg[0] - state1 = [3.0*self.loss, 3.0*self.accuracy, image_count] - state1 = state1 - state2 = [2.0*self.loss, 2.0*self.accuracy, image_count+30] - state2 = state2 - merged_state = self.subject.internal_keras_eval_merge(state1,state2) - agg_loss = merged_state[0] - agg_accuracy = merged_state[1] - image_count_total = merged_state[2] - - self.assertEqual( 2*image_count+30 , image_count_total ) - self.assertAlmostEqual( 5.0*self.loss, agg_loss, 2) - self.assertAlmostEqual( 5.0*self.accuracy, agg_accuracy, 2) - - def test_internal_keras_eval_merge_none_first(self): - image_count = self.total_images_per_seg[0] - input_state = [self.loss, self.accuracy, image_count] - merged_state = self.subject.internal_keras_eval_merge(None, input_state) - agg_loss = merged_state[0] - agg_accuracy = merged_state[1] - image_count_total = merged_state[2] - - self.assertEqual(image_count, image_count_total) - self.assertAlmostEqual(self.loss, agg_loss, 2) - self.assertAlmostEqual(self.accuracy, agg_accuracy, 2) - - def test_internal_keras_eval_merge_none_second(self): - image_count = self.total_images_per_seg[0] - input_state = [self.loss, self.accuracy, image_count] - merged_state = self.subject.internal_keras_eval_merge(input_state, None) - agg_loss = merged_state[0] - agg_accuracy = merged_state[1] - image_count_total = merged_state[2] - - self.assertEqual(image_count, image_count_total) - self.assertAlmostEqual(self.loss, agg_loss, 2) - self.assertAlmostEqual(self.accuracy, agg_accuracy, 2) - - def test_internal_keras_eval_merge_both_none(self): - result = self.subject.internal_keras_eval_merge(None,None) - self.assertEqual(None, result) - - def test_internal_keras_eval_final(self): - image_count = self.total_images_per_seg[0] - input_state = [image_count*self.loss, image_count*self.accuracy, image_count] - - output_state = self.subject.internal_keras_eval_final(input_state) - self.assertEqual(len(output_state), 2) - agg_loss = output_state[0] - agg_accuracy = output_state[1] - - self.assertAlmostEqual(self.loss, agg_loss,2) - self.assertAlmostEqual(self.accuracy, agg_accuracy,2) - - def internal_keras_eval_final_none(self): - result = self.subject.internal_keras_eval_final(None) - self.assertEqual(result, None) - - def test_internal_keras_eval_final_image_count_zero(self): - input_state = [0, 0, 0] - - with self.assertRaises(plpy.PLPYException): - result = self.subject.internal_keras_eval_final(input_state) - - def tearDown(self): - self.module_patcher.stop() - -if __name__ == '__main__': - unittest.main() -# --------------------------------------------------------------------- diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_gpu_info.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_gpu_info.py_in deleted file mode 100644 index cc885f9..0000000 --- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_gpu_info.py_in +++ /dev/null @@ -1,88 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -m4_changequote(`') - -import sys -import numpy as np -import os -from os import path -# Add dl module to the pythonpath. -sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))) -sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))) - -import unittest -from mock import * -import plpy_mock as plpy - - - -class GPUInfoTestCase(unittest.TestCase): - def setUp(self): - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import madlib_keras_gpu_info - self.module = madlib_keras_gpu_info - self.module.output_tbl_valid = Mock() - self.table_name = 'does_not_matter' - - def tearDown(self): - self.module_patcher.stop() - - def test_gpu_configuration_invalid_source(self): - with self.assertRaises(plpy.PLPYException): - self.module.gpu_configuration('schema', self.table_name, 'invalid_source') - - def test_gpu_configuration_none_source(self): - self.module.gpu_configuration('schema', self.table_name, None) - self.assertIn('tensorflow', str(self.plpy_mock_execute.call_args_list)) - self.assertNotIn('nvidia', str(self.plpy_mock_execute.call_args_list)) - - def test_gpu_configuration_tensorflow(self): - self.module.gpu_configuration('schema', self.table_name, 'tensorflow') - self.assertIn('tensorflow', str(self.plpy_mock_execute.call_args_list)) - self.assertNotIn('nvidia', str(self.plpy_mock_execute.call_args_list)) - - def test_gpu_configuration_nvidia(self): - self.module.gpu_configuration('schema', self.table_name, 'nvidia') - self.assertIn('nvidia', str(self.plpy_mock_execute.call_args_list)) - self.assertNotIn('tensorflow', str(self.plpy_mock_execute.call_args_list)) - - def test_gpu_configuration_tensorflow_uppercase(self): - self.module.gpu_configuration('schema', self.table_name, 'TENSORFLOW') - self.assertIn('tensorflow', str(self.plpy_mock_execute.call_args_list)) - self.assertNotIn('nvidia', str(self.plpy_mock_execute.call_args_list)) - - def test_gpu_configuration_nvidia_mixedcase(self): - self.module.gpu_configuration('schema', self.table_name, 'NVidIa') - self.assertIn('nvidia', str(self.plpy_mock_execute.call_args_list)) - self.assertNotIn('tensorflow', str(self.plpy_mock_execute.call_args_list)) - - -if __name__ == '__main__': - unittest.main() -# --------------------------------------------------------------------- diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in deleted file mode 100644 index 57e08a5..0000000 --- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in +++ /dev/null @@ -1,140 +0,0 @@ -# coding=utf-8 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -m4_changequote(`') - -import sys -import os -from os import path -# Add convex module to the pythonpath. -sys.path.append(path.dirname(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))))) -sys.path.append(path.dirname(path.dirname(path.dirname(path.abspath(__file__))))) - -import keras -import unittest -from mock import * -import plpy_mock as plpy - -class LoadModelSelectionTableTestCase(unittest.TestCase): - def setUp(self): - # The side effects of this class(writing to the output table) are not - # tested here. They are tested in dev-check. - self.plpy_mock = Mock(spec='error') - patches = { - 'plpy': plpy - } - - self.plpy_mock_execute = MagicMock() - plpy.execute = self.plpy_mock_execute - - self.module_patcher = patch.dict('sys.modules', patches) - self.module_patcher.start() - import deep_learning.madlib_keras_model_selection - self.module = deep_learning.madlib_keras_model_selection - self.module.MstLoaderInputValidator._validate_input_args = \ - MagicMock() - - self.subject = self.module.MstLoader - self.model_selection_table = 'mst_table' - self.model_arch_table = 'model_arch_library' - self.model_id_list = [1] - self.compile_params_list = [ - """ - loss='categorical_crossentropy', - optimizer='Adam(lr=0.1)', - metrics=['accuracy'] - """, - """ - loss='categorical_crossentropy', - optimizer='Adam(lr=0.01)', - metrics=['accuracy'] - """, - """ - loss='categorical_crossentropy', - optimizer='Adam(lr=0.001)', - metrics=['accuracy'] - """ - ] - self.fit_params_list = [ - "batch_size=5,epochs=1", - "batch_size=10,epochs=1" - ] - - def test_mst_table_dimension(self): - generate_mst = self.subject( - self.model_selection_table, - self.model_arch_table, - self.model_id_list, - self.compile_params_list, - self.fit_params_list - ) - self.assertEqual(6, len(generate_mst.msts)) - - def test_invalid_input_args(self): - self.module.MstLoaderInputValidator \ - ._validate_input_args \ - .side_effect = plpy.PLPYException('Invalid input args') - with self.assertRaises(plpy.PLPYException): - generate_mst = self.subject( - self.model_selection_table, - self.model_arch_table, - self.model_id_list, - self.compile_params_list, - self.fit_params_list - ) - - def test_duplicate_params(self): - self.model_id_list = [1, 1, 2] - self.compile_params_list = [ - """ - loss='categorical_crossentropy', - optimizer='Adam(lr=0.1)', - metrics=['accuracy'] - """, - """ - loss='categorical_crossentropy', - optimizer='Adam(lr=0.1)', - metrics=['accuracy'] - """, - """ - loss='categorical_crossentropy', - optimizer='Adam(lr=0.001)', - metrics=['accuracy'] - """ - ] - self.fit_params_list = [ - "batch_size= 5,epochs=1", - "epochs=1 ,batch_size=5", - "batch_size=10,epochs =1" - ] - generate_mst = self.subject( - self.model_selection_table, - self.model_arch_table, - self.model_id_list, - self.compile_params_list, - self.fit_params_list - ) - self.assertEqual(8, len(generate_mst.msts)) - - def tearDown(self): - self.module_patcher.stop() - -if __name__ == '__main__': - unittest.main() -# --------------------------------------------------------------------- diff --git a/src/ports/postgres/modules/elastic_net/elastic_net.py_in b/src/ports/postgres/modules/elastic_net/elastic_net.py_in index c6dec53..43c05e1 100644 --- a/src/ports/postgres/modules/elastic_net/elastic_net.py_in +++ b/src/ports/postgres/modules/elastic_net/elastic_net.py_in @@ -282,7 +282,7 @@ def elastic_net_help(schema_madlib, family_or_optimizer=None, **kwargs): def elastic_net_train(schema_madlib, source_table, model_table, dependent_varname, independent_varname, regress_family, alpha, lambda_value, standardize, grouping_col, optimizer, - optimizer_params, excluded, max_iter, tolerance, + optimizer_params, exclude, max_iter, tolerance, **kwargs): """ A wrapper for all variants of elastic net regularization. @@ -350,7 +350,7 @@ def _internal_elastic_net_train( schema_madlib, source_table, model_table, dependent_varname, independent_varname, grouping_col, regress_family, alpha, lambda_value, - standardize, optimizer, optimizer_params, excluded, + standardize, optimizer, optimizer_params, exclude, max_iter, tolerance, **kwargs): tbl_summary = add_postfix(model_table, "_summary") @@ -358,7 +358,7 @@ def _internal_elastic_net_train( # handle all special cases of independent_varname independent_varname, outstr_array = analyze_input_str( schema_madlib, source_table, - independent_varname, dependent_varname, excluded) + independent_varname, dependent_varname, exclude) # get the grouping info reserved_cols =['regress_family', 'coef_all', @@ -495,7 +495,7 @@ def _check_args(tbl_source, col_ind_var, col_dep_var): def analyze_input_str(schema_madlib, tbl_source, - col_ind_var, col_dep_var, excluded): + col_ind_var, col_dep_var, exclude): """ Make input strings and output strings compatible with functions @@ -511,7 +511,7 @@ def analyze_input_str(schema_madlib, tbl_source, col_types_dict = dict(get_cols_and_types(tbl_source)) cols = col_types_dict.keys() - s = _string_to_array(excluded) if excluded is not None else [] + s = _string_to_array(exclude) if exclude is not None else [] for each_col in cols: if each_col not in s and each_col != col_dep_var: @@ -537,7 +537,7 @@ def analyze_input_str(schema_madlib, tbl_source, if columns_exist_in_table(tbl_source, [col_ind_var], schema_madlib): # if the input is a column name and not an expression return analyze_single_input_str(schema_madlib, tbl_source, - col_ind_var, excluded) + col_ind_var, exclude) else: # if input is an expression resulting in an array output matched = is_string_formatted_as_array_expression(col_ind_var) @@ -562,7 +562,7 @@ def analyze_input_str(schema_madlib, tbl_source, def analyze_single_input_str(schema_madlib, tbl_source, col_ind_var, - excluded=None): + exclude=None): """ Args: @param schema_madlib: string, Name of schema where MADlib is installed @@ -588,8 +588,8 @@ def analyze_single_input_str(schema_madlib, tbl_source, col_ind_var, """.format(tbl_source=tbl_source, col_ind_var=col_ind_var))[0]["dimension"] - if excluded is not None: - s = _string_to_array(excluded) + if exclude is not None: + s = _string_to_array(exclude) invalid_excluded = """ Elastic Net error: When the independent variable is an array column, excluded values can only be indices diff --git a/src/ports/postgres/modules/elastic_net/elastic_net.sql_in b/src/ports/postgres/modules/elastic_net/elastic_net.sql_in index c1aaebf..2358b4e 100644 --- a/src/ports/postgres/modules/elastic_net/elastic_net.sql_in +++ b/src/ports/postgres/modules/elastic_net/elastic_net.sql_in @@ -895,7 +895,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_train ( grouping_col TEXT, optimizer TEXT, optimizer_params TEXT, - excluded TEXT, + exclude TEXT, max_iter INTEGER, tolerance DOUBLE PRECISION ) RETURNS VOID AS $$ @@ -905,7 +905,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_train ( schema_madlib, tbl_source, tbl_result, col_dep_var, col_ind_var, regress_family, alpha, lambda_value, standardize, grouping_col, optimizer, optimizer_params, - excluded, max_iter, tolerance) + exclude, max_iter, tolerance) $$ LANGUAGE plpythonu m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); @@ -923,7 +923,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_train ( grouping_columns TEXT, optimizer TEXT, optimizer_params TEXT, - excluded TEXT, + exclude TEXT, max_iter INTEGER ) RETURNS VOID AS $$ BEGIN @@ -945,7 +945,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_train ( grouping_columns TEXT, optimizer TEXT, optimizer_params TEXT, - excluded TEXT + exclude TEXT ) RETURNS VOID AS $$ BEGIN PERFORM MADLIB_SCHEMA.elastic_net_train($1, $2, $3, $4, $5, $6, $7, $8, @@ -1157,7 +1157,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_gaussian_predict ( ind_var DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', '__elastic_net_gaussian_predict' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1176,7 +1176,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_binomial_predict ( ind_var DOUBLE PRECISION[] ) RETURNS BOOLEAN AS 'MODULE_PATHNAME', '__elastic_net_binomial_predict' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1195,7 +1195,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.elastic_net_binomial_prob ( ind_var DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', '__elastic_net_binomial_prob' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1207,7 +1207,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__elastic_net_binomial_loglikelihood ( ind_var DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', '__elastic_net_binomial_loglikelihood' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1240,7 +1240,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_transition ( step_decay DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'gaussian_igd_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1250,7 +1250,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_merge ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'gaussian_igd_merge' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1259,7 +1259,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_final ( state DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'gaussian_igd_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /* @@ -1337,7 +1337,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_state_diff ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', '__gaussian_igd_state_diff' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1349,7 +1349,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_igd_result ( tolerance DOUBLE PRECISION ) RETURNS MADLIB_SCHEMA.__elastic_net_result AS 'MODULE_PATHNAME', '__gaussian_igd_result' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1372,7 +1372,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_transition ( random_stepsize INTEGER ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'gaussian_fista_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1382,7 +1382,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_merge ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'gaussian_fista_merge' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1391,7 +1391,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_final ( state DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'gaussian_fista_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /* @@ -1439,7 +1439,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_state_diff ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', '__gaussian_fista_state_diff' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1448,7 +1448,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__gaussian_fista_result ( in_state DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__elastic_net_result AS 'MODULE_PATHNAME', '__gaussian_fista_result' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1472,7 +1472,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_transition ( step_decay DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'binomial_igd_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1482,7 +1482,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_merge ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'binomial_igd_merge' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1491,7 +1491,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_final ( state DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'binomial_igd_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /* @@ -1569,7 +1569,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_state_diff ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', '__binomial_igd_state_diff' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1581,7 +1581,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_igd_result ( tolerance DOUBLE PRECISION ) RETURNS MADLIB_SCHEMA.__elastic_net_result AS 'MODULE_PATHNAME', '__binomial_igd_result' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1604,7 +1604,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_transition ( random_stepsize INTEGER ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'binomial_fista_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1614,7 +1614,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_merge ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'binomial_fista_merge' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1623,7 +1623,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_final ( state DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'binomial_fista_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /* @@ -1671,7 +1671,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_state_diff ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', '__binomial_fista_state_diff' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- @@ -1680,7 +1680,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__binomial_fista_result ( in_state DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__elastic_net_result AS 'MODULE_PATHNAME', '__binomial_fista_result' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/deep_learning/__init__.py_in b/src/ports/postgres/modules/facebook_prophet/__init__.py_in similarity index 100% rename from src/ports/postgres/modules/deep_learning/__init__.py_in rename to src/ports/postgres/modules/facebook_prophet/__init__.py_in diff --git a/src/ports/postgres/modules/facebook_prophet/prophet_alg.py_in b/src/ports/postgres/modules/facebook_prophet/prophet_alg.py_in new file mode 100644 index 0000000..a1ea25e --- /dev/null +++ b/src/ports/postgres/modules/facebook_prophet/prophet_alg.py_in @@ -0,0 +1,349 @@ +# -*- coding:utf-8 -*- +# Copyright (c) 2020 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------- +# Timeseries predict use facebook module prophet. +# ---------------------------------------------------------------------- +import plpy +import cPickle as pickle + +from utilities.validate_args import table_exists +from utilities.validate_args import table_is_empty +from utilities.validate_args import columns_exist_in_table +from utilities.validate_args import get_expr_type +from utilities.validate_args import get_cols +from utilities.utilities import _assert +from utilities.control import MinWarning + + +def prophet_fit(schema_madlib, input_table, model_table, column_map, params_str, verbose, **kwargs): + """ + Args: + @param input_table, -- name of input table + @param model_table, -- name of output table + @param column_map, -- map relation of column + @param params_str, -- names of paraters for prophet + @param verbose -- verbose or not + """ + with MinWarning('warning'): + _validate_args(schema_madlib, input_table, model_table, params_str) + + sql = """ + DROP TABLE IF EXISTS {model_table}; + CREATE TABLE {model_table} m4_ifdef(`__GSDBMPP__', `DISTRIBUTE BY HASH(id)', `') + AS + ( + SELECT + 1 as id, + (mdl_results).y_type, + (mdl_results).model, + (mdl_results).params + FROM ( + SELECT + {schema_madlib}.__get_fbmodel_as_bytea__( + '{input_table}', + $${column_map}$$, + $${params_str}$$, + {verbose} + ) as mdl_results + )q + ); + """.format( + schema_madlib=schema_madlib, + model_table=model_table, + input_table=input_table, + column_map=column_map, + params_str=params_str, + verbose=verbose + ) + + if verbose: + plpy.info(sql) + plpy.execute(sql) + + return "Prophet_fit over and model is saved in {model_table}".format(model_table=model_table) + + +# ------------------------------------------------------------------------------ +# -- prophet predict function ------------------------------------------------------------ +# ------------------------------------------------------------------------------ +def prophet_predict(schema_madlib, model_table, output_table, periods, freq, cap, floor, include_history, verbose, **kwargs): + """ + Args: + @param model_table, -- name of model table + @param output_table, -- name of output table + @param periods, -- value of predict length + @param freq, -- frequency of predict value + @param cap, -- max value of predict value(logistic) + @param floor, -- min value of predict value(logistic) + @param inlcude_history + """ + with MinWarning('warning'): + _validate_predict(schema_madlib, model_table, output_table, periods) + + sql = """select y_type, model from {model_table};""".format(model_table=model_table) + if verbose: + plpy.info(sql) + result = plpy.execute(sql) + y_type = result[0]['y_type'] + + model = result[0]['model'] + model = pickle.loads(model) + + if freq: + future = model.make_future_dataframe(periods=periods, freq=freq, include_history=include_history) + else: + future = model.make_future_dataframe(periods=periods, include_history=include_history) + if cap is not None: + future['cap'] = cap + if floor is not None: + future['floor'] = floor + forecast = model.predict(future) + + columns = forecast.columns + column_list, value_list = [], [] + for column in columns: + if column == 'ds': + column_list.append('{column} date'.format(column=column)) + value_list.append("'{{{column}}}'".format(column=column)) + else: + column_list.append('{column} {y_type}'.format(column=column, y_type=y_type)) + value_list.append('{{{column}}}'.format(column=column)) + column_information = ', '.join(column_list) + value_information = ', '.join(value_list) + plpy.info("column_information is: " + str(column_information)) + plpy.info("value_information is: " + str(value_information)) + + sql = """ + DROP TABLE IF EXISTS {output_table}; + CREATE TABLE {output_table} ({column_information}) + """.format( + output_table=output_table, + column_information=column_information + ) + if verbose: + plpy.info(sql) + plpy.execute(sql) + + + sql = """INSERT INTO {output_table} VALUES ({value_information});""".format(output_table=output_table, + value_information=value_information) + plpy.info("insert sql is {sql}".format(sql=sql)) + predict_length = len(forecast['ds']) + for i in range(predict_length): + plpy.execute( + sql.format(**{col:forecast[col][i] for col in columns}) + ) + + return "prophet predict over and result is saved in {output_table}".format(output_table=output_table) + + +# ------------------------------------------------------------------------------ +# -- validate args ------------------------------------------------------------ +# ------------------------------------------------------------------------------ +def _validate_args(schema_madlib, input_table, model_table, params_str): + """ + @brief validate the arguments + """ + # 1, check input_table: name. + _assert(input_table and input_table.strip().lower() not in ('null', ''), + "Prophet error: Invalid data table name!") + _assert(table_exists(input_table), + "Prophet error: Data table does not exist!") + _assert(not table_is_empty(input_table), + "Prophet error: Data table is empty!") + + # 3, check output table. + _assert(model_table is not None and model_table.strip().lower() not in ('null', ''), + "Prophet error: Invalid output table name!") + _assert(not table_exists(model_table, only_first_schema=True), + "Output table name already exists. Drop the table before calling the function.") + + # 4, check params_str + if params_str != '': + # let prophet to check the parameter + pass + + +def _validate_predict(schema_madlib, model_table, output_table, periods): + """ + @brief validate the predict arguments + """ + # 1, check input_table: name. + _assert(model_table and model_table.strip().lower() not in ('null', ''), + "Prophet error: Invalid model table name!") + _assert(table_exists(model_table), + "Prophet error: model table does not exist!") + _assert(not table_is_empty(model_table), + "Prophet error: Data table is empty!") + + # 3, check output table. + _assert(output_table is not None and output_table.strip().lower() not in ('null', ''), + "Prophet error: Invalid output_table table name!") + _assert(not table_exists(output_table, only_first_schema=True), + "Output table name already exists. Drop the table before calling the function.") + + # 4, check periods + _assert(isinstance(periods, int) and periods > 0, "Prophet error: Invalid parameter 'periods'") + + +# ------------------------------------------------------------------------------ +# -- Online help function ------------------------------------------------------ +# ------------------------------------------------------------------------------ +def prophet_fit_help_message(schema_madlib, message, **kwargs): + """ Help message for Prophet_fit + + @brief + Args: + @param schema_madlib string, Name of the schema madlib + @param message string, Help message indicator + + Returns: + String. Contain the help message string + """ + if not message: + help_string=""" + ----------------------------------------------------------------------- + SUMMARY + ----------------------------------------------------------------------- + facebook prophet API timeseries predict. + + For more details on function usage: + SELECT {schema_madlib}.prophet_fit('usage') + """ + elif message in ['usage', 'help', '?']: + help_string=""" + ----------------------------------------------------------------------- + USAGE + ----------------------------------------------------------------------- + SELECT {schema_madlib}.prophet_fit( + input_table TEXT, -- in, Data table name + model_table TEXT, -- out, save model and features + column_map TEXT, -- map relation of column : 'ds', 'y', 'floor', 'cap' + params_str TEXT, -- prophet parameters + verbose BOOLEAN -- Boolean, whether to print more info, default is False + ); + + ----------------------------------------------------------------------- + parameters + ----------------------------------------------------------------------- + below is optional parameters: + + column_map: + Prophet has specified column name as follows: + 'ds': column of date + 'y': column of predict value + 'cap': upper limit of predict value + 'floor': lower limit of predict value + + so in order to simplify the usage of prophet, we allow user to provide a map relationship between + specified column name and corresponding column name, for example: + date | value + ------------------ + 2000-1-1 | 100 + 2000-1-2 | 120 + 2000-1-3 | 130 + + in this case, we should provide a map like {{'ds': 'date', 'y': 'value'}} + if user do not provide map relationship, then use the default {{'ds': 'ds', 'y': 'y'}}, if you want to use 'cap' and 'floor', + you should definite it in column_map, otherwise will not use. + + params_str: + For example, if the input is: + $${{'growth': 'linear', 'changepoints': ['2000-01-01', '2000-02-02'], 'yearly_seasonality': True}}$$ + + a full input query: + select {schema_madlib}.prophet_fit('inputTable', 'output_modelTable', $${{'ds':'ds', 'y': 'y', 'cap': 'cap'}}$$, + $${{'growth': 'linear', 'changepoints': ['2000-01-01', '2000-02-02'], 'yearly_seasonality': True, 'holidays': ['playoffs']}}$$, True); + + ----------------------------------------------------------------------- + OUTPUT + ----------------------------------------------------------------------- + The output table ('model_table' above) has the following columns: + id -- index + model -- binary model + + note: the type of ds_column must be DATE + + """ + else: + help_string="No such option. Use {schema_madlib}.prophet_fit()" + + return help_string.format(schema_madlib=schema_madlib) + + +def prophet_predict_help_message(schema_madlib, message, **kwargs): + """ Help message for Prophet_predict + + @brief + Args: + @param schema_madlib string, Name of the schema madlib + @param message string, Help message indicator + + Returns: + String. Contain the help message string + """ + if not message: + help_string=""" + ----------------------------------------------------------------------- + SUMMARY + ----------------------------------------------------------------------- + facebook prophet API timeseries predict. + + For more details on function usage: + SELECT {schema_madlib}.prophet_predict('usage') + """ + elif message in ['usage', 'help', '?']: + help_string=""" + ----------------------------------------------------------------------- + USAGE + ----------------------------------------------------------------------- + SELECT {schema_madlib}.prophet_predict( + model_table TEXT, -- in, model table name + output_table TEXT, -- out, save predict result + periods INTEGER, -- the number of periods to forecast forward. + freq TEXT, -- 'day', 'week', 'month', 'quarter', 'year', 1(1 sec), 60(1 minute) or 3600(1 hour) + cap DOUBLE PRECISION, -- the max value in future predict(logistic mode) + floor DOUBLE PRECISION, -- the min value in future predict(logistic mode) + include_history BOOLEAN, -- to include the historical dates in the data frame for predictions. + verbose BOOLEAN -- whether to print more info, default is False + ); + + ----------------------------------------------------------------------- + parameters + ----------------------------------------------------------------------- + below is optional parameters: + + periods: + periods >= 1 + cap, floor: + if growth is 'logistic', user should provide 'cap' or 'floor', if do not use it, just set NULL. + + example: select {schema_madlib}.prophet_predict('modelTable', 'output_Table', 1000, '10T', 10, NULL, False, False); + example: select {schema_madlib}.prophet_predict('modelTable', 'output_Table', 1000, '10T', NULL, NULL, False, False); + + + ----------------------------------------------------------------------- + OUTPUT + ----------------------------------------------------------------------- + The output table include all result of prophet_predict + """ + else: + help_string="No such option. Use {schema_madlib}.prophet_fit()" + + return help_string.format(schema_madlib=schema_madlib) + diff --git a/src/ports/postgres/modules/facebook_prophet/prophet_alg.sql_in b/src/ports/postgres/modules/facebook_prophet/prophet_alg.sql_in new file mode 100644 index 0000000..9bace08 --- /dev/null +++ b/src/ports/postgres/modules/facebook_prophet/prophet_alg.sql_in @@ -0,0 +1,186 @@ +/* + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + */ + +------------------------------------------------------------------------------------------------------------------------------- +-- Build facebook Prophet models in-database - - +------------------------------------------------------------------------------------------------------------------------------- +-- +----------- +-- Note: -- +----------- +-- 1) Porphet uses facebook-API. However we're limited by the maximum +-- field-size in Greenplum/Postgres which is currently 1 GB. +-- 2) If your dataset is much larger (> 1 GB), it is strongly recommended that you use MADlib's models so that +-- training & scoring will happen in-parallel on all nodes. +------------------------------------------------------------------------------------------------------------------------------- + +m4_include(`SQLCommon.m4') + +---------------------------------------------------------------------------------------------- +------ Help messages +---------------------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit( + message TEXT +) RETURNS TEXT AS $XX$ + PythonFunction(facebook_prophet, prophet_alg, prophet_fit_help_message) +$XX$ LANGUAGE plpythonu IMMUTABLE; +------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit() +RETURNS TEXT AS $XX$ + SELECT MADLIB_SCHEMA.prophet_fit(''::TEXT); +$XX$ +LANGUAGE sql IMMUTABLE; + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict( + message TEXT +) RETURNS TEXT AS $XX$ + PythonFunction(facebook_prophet, prophet_alg, prophet_predict_help_message) +$XX$ LANGUAGE plpythonu IMMUTABLE; +------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict() +RETURNS TEXT AS $XX$ + SELECT MADLIB_SCHEMA.prophet_predict(''::TEXT); +$XX$ +LANGUAGE sql IMMUTABLE; + + +---------------------------------------------------------------------------------------------- +------ prophet fit +---------------------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit( + input_table TEXT, -- in, dataset table + model_table TEXT, -- out, save model and features + column_map TEXT, -- map relation of column {'ds' 'y' 'floor' 'cap'} + params_str TEXT, -- prophet parameters + verbose BOOLEAN +) +RETURNS TEXT AS $$ +PythonFunction(facebook_prophet, prophet_alg, prophet_fit) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit( + input_table TEXT, + model_table TEXT, + column_map TEXT, + params_str TEXT +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.prophet_fit($1, $2, $3, $4, False); +$$ LANGUAGE sql; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit( + input_table TEXT, + model_table TEXT, + column_map TEXT + +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.prophet_fit($1, $2, $3, NULL); +$$ LANGUAGE sql; + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_fit( + input_table TEXT, + model_table TEXT + +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.prophet_fit($1, $2, NULL); +$$ LANGUAGE sql; + + +--def prophet_predict(schema_madlib, model_table, output_table, periods, verbose, **kwargs): + + +---------------------------------------------------------------------------------------------- +------ prophet predict +---------------------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict( + model_table TEXT, -- in, dataset table + output_table TEXT, -- out, save model and features + periods INTEGER, -- predict length + freq TEXT, -- frequency of input_dataset + cap DOUBLE PRECISION, -- max value of predict value(logistic) + floor DOUBLE PRECISION, -- min value of predict value(logistic) + include_history BOOLEAN, -- include history dataset in output_table + verbose BOOLEAN -- log or not +) +RETURNS TEXT AS $$ +PythonFunction(facebook_prophet, prophet_alg, prophet_predict) +$$ LANGUAGE plpythonu; + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict( + model_table TEXT, -- in, dataset table + output_table TEXT, -- out, save model and features + periods INTEGER, -- predict length + freq TEXT, -- frequency of input_dataset + cap DOUBLE PRECISION, -- max value of predict value(logistic) + floor DOUBLE PRECISION, -- min value of predict value(logistic) + include_history BOOLEAN -- include history dataset in output_table +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, $4, $5, $6, $7, False); +$$ LANGUAGE sql; + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict( + model_table TEXT, -- in, dataset table + output_table TEXT, -- out, save model and features + periods INTEGER, -- predict length + freq TEXT, -- frequency of input_dataset + cap DOUBLE PRECISION, -- max value of predict value(logistic) + floor DOUBLE PRECISION -- min value of predict value(logistic) +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, $4, $5, $6, False); +$$ LANGUAGE sql; + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict( + model_table TEXT, -- in, dataset table + output_table TEXT, -- out, save model and features + periods INTEGER, -- predict length + freq TEXT, -- frequency of input_dataset + cap DOUBLE PRECISION -- max value of predict value(logistic) +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, $4, $5, NULL); +$$ LANGUAGE sql; + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict( + model_table TEXT, -- in, dataset table + output_table TEXT, -- out, save model and features + periods INTEGER, -- predict length + freq TEXT -- frequency of input_dataset +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, $4, NULL); +$$ LANGUAGE sql; + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.prophet_predict( + model_table TEXT, -- in, dataset table + output_table TEXT, -- out, save model and features + periods INTEGER -- predict length +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.prophet_predict($1, $2, $3, NULL); +$$ LANGUAGE sql; + + diff --git a/src/ports/postgres/modules/facebook_prophet/prophet_utilities.py_in b/src/ports/postgres/modules/facebook_prophet/prophet_utilities.py_in new file mode 100644 index 0000000..2eaba82 --- /dev/null +++ b/src/ports/postgres/modules/facebook_prophet/prophet_utilities.py_in @@ -0,0 +1,107 @@ +# -*- coding:utf-8 -*- +# Copyright (c) 2020 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- + +import re +import ast +import plpy + +import pandas as pd +import cPickle as pickle +from fbprophet import Prophet + +from utilities.validate_args import get_expr_type + + +support_type_list = ('TINYINT', 'SMALLINT', 'INTEGER', 'BINARY_INTEGER', 'BIGINT', + 'NUMERIC', 'DECIMAL', + 'SMALLSERIAL', 'SERIAL', 'BIGSERIAL', + 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'FLOAT8', 'FLOAT', 'BINARY_DOUBLE', 'DEC', + 'BOOLEAN') + + +def get_fbmodel(schema_madlib, input_table, column_map, params_str, verbose, **kwargs): + """ + train prophet model and return as bytea + Args: + @param input_table, -- name of input table + @param column_map, -- map relationship of specified name of column and current name + @param params_str, -- paramater of prophet fit function + @param verbose -- display log + """ + if params_str is None or params_str is "" or ":" not in params_str: + params_str = "{}" + plpy.info("prophet_fit() params_str type is: " + params_str) + + if column_map is None or column_map is "" or ":" not in column_map: + column_map = "{'ds': 'ds', 'y': 'y'}" + plpy.info("Prophet_fit() column_map relation is: " + column_map) + + params = ast.literal_eval(re.sub("[\\t]", "", params_str).strip()) + column_map_dict = ast.literal_eval(re.sub("[\\t]", "", column_map).strip()) + + y_type = check_input_type(schema_madlib, input_table, column_map_dict) + + if 'holidays' in params.keys(): + holidays = pd.DataFrame() + holiday_tables = params['holidays'] + sql = """select * from {holiday_table}""" + for holiday_table in holiday_tables: + sql = sql.format(holiday_table=holiday_table) + if verbose: + plpy.info(sql) + result = plpy.execute(sql) + holiday_df = pd.DataFrame.from_records(result) + holidays = pd.concat((holidays, holiday_df)) + params['holidays'] = holidays + + real_columns = ', '.join([value for value in column_map_dict.values()]) + sql = """select {real_columns} from {input_table};""".format(input_table=input_table, real_columns=real_columns) + if verbose: + plpy.info(sql) + + result = plpy.execute(sql) + df = pd.DataFrame.from_records(result) + + # change column name to accord with prophet requirement + for s_name, c_name in column_map_dict.items(): + if s_name != c_name: + df.rename(columns={c_name: s_name}, inplace=True) + model = Prophet(**params) + model.fit(df) + + return (y_type, pickle.dumps(model), str(params)) + + +# ------------------------------------------------------------------------------ +# -- validate column type of input table ------------------------------------------------------------ +# ------------------------------------------------------------------------------ +def check_input_type(schema_madlib, input_table, column_map_dict): + """ + func: check the column type of input_table + """ + for s_col, c_col in column_map_dict.items(): + col_type = get_expr_type(c_col, input_table) + if s_col == 'ds': + if 'time' not in col_type.lower() and 'date' not in col_type.lower(): + plpy.error("prophet Error: the type of {column} must be DATE".format(column=c_col)) + else: + if col_type.upper() not in support_type_list: + plpy.error("prophet Error: the type of {column} is not supported".format(column=c_col)) + + y_type = get_expr_type(column_map_dict['y'], input_table) + + return y_type diff --git a/src/ports/postgres/modules/facebook_prophet/prophet_utilities.sql_in b/src/ports/postgres/modules/facebook_prophet/prophet_utilities.sql_in new file mode 100644 index 0000000..4a132b4 --- /dev/null +++ b/src/ports/postgres/modules/facebook_prophet/prophet_utilities.sql_in @@ -0,0 +1,54 @@ +/* + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + */ + +------------------------------------------------------------------------------------------------------------------------------- +-- Build facebook Prophet models in-database - - +------------------------------------------------------------------------------------------------------------------------------- +-- +----------- +-- Note: -- +----------- +-- 1) Porphet uses facebook-API. However we're limited by the maximum +-- field-size in Greenplum/Postgres which is currently 1 GB. +-- 2) If your dataset is much larger (> 1 GB), it is strongly recommended that you use MADlib's models so that +-- training & scoring will happen in-parallel on all nodes. +------------------------------------------------------------------------------------------------------------------------------- + +m4_include(`SQLCommon.m4') + + +------------------------------------------------ +-- Type for internal train results +------------------------------------------------ +DROP TYPE IF EXISTS MADLIB_SCHEMA.prophet_fit_results_type cascade; +CREATE TYPE MADLIB_SCHEMA.prophet_fit_results_type AS( + y_type TEXT, -- save X + model bytea, -- dump model + params TEXT -- parameters used +); + + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__get_fbmodel_as_bytea__( + input_table TEXT, -- dataset table + column_map TEXT, -- map relation of column {'ds' 'y' 'floor' 'cap'} + params_str TEXT, -- xgboost parameters + verbose BOOLEAN +) +RETURNS MADLIB_SCHEMA.prophet_fit_results_type AS $$ +PythonFunction(facebook_prophet, prophet_utilities, get_fbmodel) +$$ LANGUAGE plpythonu; + diff --git a/src/ports/postgres/modules/facebook_prophet/test/prophet_test.sql_in b/src/ports/postgres/modules/facebook_prophet/test/prophet_test.sql_in new file mode 100644 index 0000000..37b8c88 --- /dev/null +++ b/src/ports/postgres/modules/facebook_prophet/test/prophet_test.sql_in @@ -0,0 +1,1522 @@ +/* +* Test case of facebook-prophet +*/ + +m4_include(`SQLCommon.m4') + +select MADLIB_SCHEMA.prophet_fit(); +select MADLIB_SCHEMA.prophet_fit('usage'); + +DROP TABLE IF EXISTS ts_data; +CREATE TABLE ts_data (date date, value float); + + +INSERT INTO ts_data (date, value) VALUES ('2016-11-29 21:20:00', 5.592186415908717);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 21:30:00', 5.213219014779573); +INSERT INTO ts_data (date, value) VALUES ('2016-11-29 21:40:00', 5.349837267365468);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 21:50:00', 5.348009861982742); +INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:00:00', 5.141486962759949);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:10:00', 5.831837644472339); +INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:20:00', 5.586104738236943);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:30:00', 5.424038616617589); +INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:40:00', 5.3953055294925);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 22:50:00', 5.08062402180196); +INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:00:00', 5.216144429037251);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:10:00', 5.89720541594696); +INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:20:00', 5.903379069573093);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:30:00', 5.108443063954224); +INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:40:00', 5.829479550836606);INSERT INTO ts_data (date, value) VALUES ('2016-11-29 23:50:00', 5.983691151745764); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:00:00', 5.911012559550056);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:10:00', 5.267628873207424); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:20:00', 5.440281847906313);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:30:00', 5.056752106326658); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:40:00', 5.57695918188404);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 00:50:00', 5.708959203703136); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:00:00', 5.837530980321919);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:10:00', 5.397656757637807); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:20:00', 5.80272939233592);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:30:00', 5.05404686767896); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:40:00', 5.599526207554924);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 01:50:00', 5.624649182696436); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:00:00', 5.630874726351715);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:10:00', 5.885412123046953); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:20:00', 5.7281711367203965);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:30:00', 5.367035247234945); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:40:00', 5.647728469736866);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 02:50:00', 5.422412855958562); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:00:00', 5.084810422982988);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:10:00', 5.045434995298759); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:20:00', 5.903595563281863);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:30:00', 5.8464706036749226); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:40:00', 5.423780748843331);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 03:50:00', 5.698599390020942); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:00:00', 5.620828657830448);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:10:00', 5.853793611596745); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:20:00', 5.1160341760917465);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:30:00', 5.760091737386397); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:40:00', 5.547488495151962);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 04:50:00', 5.074259195264116); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:00:00', 5.767149486416109);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:10:00', 5.5412954545856525); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:20:00', 5.671416922868077);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:30:00', 5.214281328444059); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:40:00', 5.704061227110594);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 05:50:00', 5.981904654103126); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:00:00', 5.811672256762379);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:10:00', 5.636629064171528); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:20:00', 5.222889294671177);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:30:00', 5.7944970498250745); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:40:00', 5.301092523899824);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 06:50:00', 5.378620727476704); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:00:00', 5.832205512624812);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:10:00', 5.1940020098503785); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:20:00', 5.2801063135872415);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:30:00', 5.333104829752228); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:40:00', 5.845933760530428);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 07:50:00', 5.932520038392692); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:00:00', 5.64679296474034);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:10:00', 5.209244473298587); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:20:00', 5.379575393446499);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:30:00', 5.551810996707294); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:40:00', 5.98125647954102);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 08:50:00', 5.352159243180884); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:00:00', 5.98297448186513);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:10:00', 5.05840496515969); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:20:00', 5.071467132599093);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:30:00', 5.508013094801815); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:40:00', 5.593629750578956);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 09:50:00', 5.0073554230453405); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:00:00', 5.08721378732741);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:10:00', 5.659885556354787); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:20:00', 5.422996896584455);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:30:00', 5.430291377769443); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:40:00', 5.661941353506331);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 10:50:00', 5.222650073421543); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:00:00', 5.422560019488827);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:10:00', 5.2976478671030405); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:20:00', 5.584338761217304);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:30:00', 5.028576011172816); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:40:00', 5.247034856187782);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 11:50:00', 5.229344731201686); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:00:00', 5.471182531178972);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:10:00', 5.0949254922245055); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:20:00', 5.683636089958879);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:30:00', 5.437006581310596); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:40:00', 5.245413304985197);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 12:50:00', 5.522111051119028); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:00:00', 5.005599900215963);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:10:00', 5.53702445367031); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:20:00', 5.5766246582094405);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:30:00', 5.258581727612666); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:40:00', 5.464358328063363);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 13:50:00', 5.91011805664996); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:00:00', 10.88382409798199);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:10:00', 10.555427090136963); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:20:00', 10.2695546171855);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:30:00', 10.96434493248081); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:40:00', 10.005061878030595);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 14:50:00', 10.053893326397173); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:00:00', 10.207155270283303);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:10:00', 10.217068622058886); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:20:00', 10.2800096297076);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:30:00', 10.084258019727397); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:40:00', 10.870604507956386);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 15:50:00', 10.139519698694448); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:00:00', 10.966851433652065);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:10:00', 10.208452839253603); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:20:00', 10.677742584132282);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:30:00', 10.20991792799135); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:40:00', 10.243176260100649);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 16:50:00', 10.24725814854844); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:00:00', 10.76947018545776);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:10:00', 10.637158701289039); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:20:00', 10.531066872723857);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:30:00', 10.699064458359702); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:40:00', 10.857099725561811);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 17:50:00', 10.947585602968706); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:00:00', 10.072388396154338);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:10:00', 10.269329386566012); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:20:00', 10.100058130861127);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:30:00', 10.630305282558199); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:40:00', 10.276992479817986);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 18:50:00', 10.823962547747168); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:00:00', 10.938196297447686);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:10:00', 10.751352407863825); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:20:00', 10.55267731888166);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:30:00', 10.95764805135116); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:40:00', 10.319600419308607);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 19:50:00', 10.894597987994707); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:00:00', 10.561011949231316);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:10:00', 10.558376344084758); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:20:00', 10.54238813342208);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:30:00', 10.353533152566689); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:40:00', 10.94670826781977);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 20:50:00', 10.867785908808944); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:00:00', 10.743813978598242);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:10:00', 10.553624735869224); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:20:00', 10.48483757931147);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:30:00', 10.842613332127966); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:40:00', 10.381878265543788);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 21:50:00', 10.044634020757982); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:00:00', 10.606953617509358);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:10:00', 10.63043787557256); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:20:00', 10.558000303473458);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:30:00', 10.119618312591788); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:40:00', 10.420324078784883);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 22:50:00', 10.831853130351744); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:00:00', 10.383033393248413);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:10:00', 10.630842365846464); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:20:00', 10.076453778579774);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:30:00', 10.228368345858518); +INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:40:00', 10.636636225999974);INSERT INTO ts_data (date, value) VALUES ('2016-11-30 23:50:00', 10.809387112125291); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:00:00', 10.555836272306514);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:10:00', 10.17361788915921); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:20:00', 10.935490280962807);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:30:00', 10.336876569080902); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:40:00', 10.315564948016368);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 00:50:00', 10.0503251516345); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:00:00', 10.686498994079804);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:10:00', 10.488270765930583); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:20:00', 10.382238780237266);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:30:00', 10.664282558119293); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:40:00', 10.482673868507927);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 01:50:00', 10.690901645112259); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:00:00', 10.808960869817462);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:10:00', 10.892103395445046); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:20:00', 10.889652181902528);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:30:00', 10.087235381673107); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:40:00', 10.354527252749081);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 02:50:00', 10.704626831711339); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:00:00', 10.653290677771649);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:10:00', 10.527049427857113); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:20:00', 10.193508167768368);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:30:00', 10.192935509184334); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:40:00', 10.807842757555397);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 03:50:00', 10.224608561871785); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:00:00', 10.926938435835226);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:10:00', 10.432633335671081); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:20:00', 10.640811656424226);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:30:00', 10.99419293027347); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:40:00', 10.397638983480087);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 04:50:00', 10.284742522103349); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:00:00', 10.74937007324194);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:10:00', 10.643810271386382); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:20:00', 10.937587864934304);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:30:00', 10.996094208628847); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:40:00', 10.933699502883464);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 05:50:00', 10.009530363086641); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:00:00', 10.753814613452027);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:10:00', 10.044290103015088); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:20:00', 10.112825899390046);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:30:00', 10.466639947711943); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:40:00', 15.453979216734295);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 06:50:00', 15.733941145738592); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:00:00', 15.101733596739034);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:10:00', 15.649935609313715); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:20:00', 15.541164284639525);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:30:00', 15.42338136917619); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:40:00', 15.731067751045787);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 07:50:00', 15.580468045277195); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:00:00', 15.26256456459248);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:10:00', 15.644521316946905); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:20:00', 15.118407201560444);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:30:00', 15.635310837394789); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:40:00', 15.914518468164172);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 08:50:00', 15.987868390442582); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:00:00', 15.42341614964402);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:10:00', 15.034129189040149); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:20:00', 15.025541704618867);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:30:00', 15.418416247429313); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:40:00', 15.861698475422092);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 09:50:00', 15.62331366505197); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:00:00', 15.720846440190945);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:10:00', 15.445679787641966); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:20:00', 15.159921163629136);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:30:00', 15.239201767432013); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:40:00', 15.787583155910015);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 10:50:00', 15.368527703049724); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:00:00', 15.966288763031649);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:10:00', 15.932848769581149); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:20:00', 15.12096784657616);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:30:00', 15.044036821254055); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:40:00', 15.031949588422053);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 11:50:00', 15.360696548582007); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:00:00', 15.548304448740831);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:10:00', 15.321778753166079); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:20:00', 15.957690239835765);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:30:00', 15.135722181550765); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:40:00', 15.480346629237996);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 12:50:00', 15.968521145540633); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:00:00', 15.747070467680564);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:10:00', 15.894977279535198); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:20:00', 15.43725890131943);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:30:00', 15.29963684376817); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:40:00', 15.949624102437316);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 13:50:00', 15.78710097386125); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:00:00', 15.383243943979766);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:10:00', 15.928153538177432); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:20:00', 15.301232004415052);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:30:00', 15.993251015437984); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:40:00', 15.508285598925116);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 14:50:00', 15.013755434803087); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:00:00', 15.075152055184526);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:10:00', 15.984582015476505); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:20:00', 15.790632765623062);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:30:00', 15.934347484557644); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:40:00', 15.444429332540322);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 15:50:00', 15.086377880107932); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:00:00', 15.811316282743125);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:10:00', 15.178644553055447); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:20:00', 15.411280626267777);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:30:00', 15.847018277776037); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:40:00', 15.784696373701456);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 16:50:00', 15.128075902863381); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:00:00', 15.344736721321144);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:10:00', 15.61790225217054); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:20:00', 15.274615030013528);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:30:00', 15.808473672968695); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:40:00', 15.020708990567766);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 17:50:00', 15.30059613604699); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:00:00', 15.498659236830926);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:10:00', 15.387232492871144); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:20:00', 15.323700631336765);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:30:00', 15.806051242162896); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:40:00', 15.247407375461128);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 18:50:00', 15.901856510495984); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:00:00', 15.439721502108116);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:10:00', 15.289082346518994); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:20:00', 15.059525497475034);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:30:00', 15.270817224386326); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:40:00', 15.871151199428525);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 19:50:00', 15.282120255097865); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:00:00', 15.316159525858552);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:10:00', 15.152434228608037); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:20:00', 15.040599454382349);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:30:00', 15.327120128690266); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:40:00', 15.113808122667509);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 20:50:00', 15.108852157293637); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:00:00', 15.601574694793966);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:10:00', 15.765348827942226); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:20:00', 15.44357776208238);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:30:00', 15.21025279747494); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:40:00', 15.998788041370798);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 21:50:00', 15.531252123355971); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:00:00', 15.400264196778496);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:10:00', 15.659934846250227); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:20:00', 15.254682422239783);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:30:00', 15.881353351807892); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:40:00', 15.943747331476715);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 22:50:00', 15.234881305321348); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:00:00', 15.830517028216903);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:10:00', 15.93265269144619); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:20:00', 20.852791006475595);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:30:00', 20.35657131996931); +INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:40:00', 20.32344079327797);INSERT INTO ts_data (date, value) VALUES ('2016-12-01 23:50:00', 20.09582415408); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:00:00', 20.733996132413107);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:10:00', 20.68778370171752); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:20:00', 20.467058725004406);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:30:00', 20.379565806425727); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:40:00', 20.42779269193495);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 00:50:00', 20.094034979346922); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:00:00', 20.227793103797048);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:10:00', 20.936411620696283); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:20:00', 20.619209447538687);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:30:00', 20.035610633780962); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:40:00', 20.445424163695513);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 01:50:00', 20.206654001756675); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:00:00', 20.55722279943204);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:10:00', 20.41050994653755); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:20:00', 20.549783559583858);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:30:00', 20.446097907452973); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:40:00', 20.48449618191396);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 02:50:00', 20.747795682422506); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:00:00', 20.16175127009515);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:10:00', 20.186956399208864); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:20:00', 20.109471316760608);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:30:00', 20.501471852177858); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:40:00', 20.46868658303998);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 03:50:00', 20.039331198435775); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:00:00', 20.733510062349858);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:10:00', 20.77518818818196); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:20:00', 20.593513060310155);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:30:00', 20.37800130713625); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:40:00', 20.51535234903095);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 04:50:00', 20.799764794377783); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:00:00', 20.05562466198948);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:10:00', 20.918634517305108); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:20:00', 20.46738766522704);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:30:00', 20.410966096668435); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:40:00', 20.20109304019048);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 05:50:00', 20.425794443629037); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:00:00', 20.764648965476564);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:10:00', 20.73638437971804); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:20:00', 20.86784935806068);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:30:00', 20.106516840699392); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:40:00', 20.26814388324929);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 06:50:00', 20.15582088751805); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:00:00', 20.365411463552753);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:10:00', 20.68406685337937); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:20:00', 20.43711892282266);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:30:00', 20.80948770424489); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:40:00', 20.76846628710522);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 07:50:00', 20.116343590304385); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:00:00', 20.31241560439532);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:10:00', 20.69080441002456); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:20:00', 20.90764764555535);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:30:00', 20.975810293040016); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:40:00', 20.17280945563313);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 08:50:00', 20.538743400676477); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:00:00', 20.237998746198095);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:10:00', 20.79471167783884); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:20:00', 20.938577199072107);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:30:00', 20.501043185915197); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:40:00', 20.87298186827212);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 09:50:00', 20.70039747550453); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:00:00', 20.330010498371948);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:10:00', 20.965259301979472); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:20:00', 20.463913248270238);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:30:00', 20.335555892985514); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:40:00', 20.225981568350765);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 10:50:00', 20.328455432029735); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:00:00', 20.409318013480828);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:10:00', 20.433148262692203); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:20:00', 20.96312507182427);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:30:00', 20.27135526020552); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:40:00', 20.25937747505829);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 11:50:00', 20.86249812853205); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:00:00', 20.766973613848872);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:10:00', 20.922823389181865); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:20:00', 20.696072941144376);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:30:00', 20.742725299116763); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:40:00', 20.00019076674507);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 12:50:00', 20.066985912811532); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:00:00', 20.66343326593504);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:10:00', 20.969064689272464); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:20:00', 20.62983584739398);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:30:00', 20.2379671764757); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:40:00', 20.579703611822953);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 13:50:00', 20.915224922908354); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:00:00', 20.05043609646298);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:10:00', 20.360897759018673); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:20:00', 20.90506269170392);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:30:00', 20.4103630269194); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:40:00', 20.287250690598615);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 14:50:00', 20.215319266294166); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:00:00', 20.81765612016418);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:10:00', 20.771935624088517); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:20:00', 20.944209556384703);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:30:00', 20.610037946505567); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:40:00', 20.81389531856648);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 15:50:00', 20.195946366739108); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:00:00', 25.66624128668068);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:10:00', 25.391982132308694); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:20:00', 25.84787871226652);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:30:00', 25.600605742306875); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:40:00', 25.35636326043316);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 16:50:00', 25.225735066913); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:00:00', 25.359039779493045);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:10:00', 25.012426558020447); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:20:00', 25.82713770767008);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:30:00', 25.442540638926115); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:40:00', 25.274139131778046);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 17:50:00', 25.35556174613059); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:00:00', 25.75836325601163);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:10:00', 25.83775033925234); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:20:00', 25.742874700634285);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:30:00', 25.77207446486909); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:40:00', 25.803784200896395);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 18:50:00', 25.214425678018586); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:00:00', 25.538252382613273);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:10:00', 25.98189905172387); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:20:00', 25.815837254882865);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:30:00', 25.252159772850863); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:40:00', 25.158702508825808);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 19:50:00', 25.856533824655532); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:00:00', 25.72820949392227);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:10:00', 25.66981889764483); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:20:00', 25.459337912377826);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:30:00', 25.581434405890104); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:40:00', 25.915241183791146);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 20:50:00', 25.69607035802048); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:00:00', 25.589296120119702);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:10:00', 25.126513844500145); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:20:00', 25.800878550322214);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:30:00', 25.755925805605784); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:40:00', 25.857677764869447);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 21:50:00', 25.729061788223557); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:00:00', 25.028572810770456);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:10:00', 25.52433676911993); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:20:00', 25.689712420391746);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:30:00', 25.913576980992943); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:40:00', 25.95352067181972);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 22:50:00', 25.72743288406616); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:00:00', 25.207218028481336);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:10:00', 25.859512492495718); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:20:00', 25.914536506877173);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:30:00', 25.29069846726668); +INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:40:00', 25.999551138048073);INSERT INTO ts_data (date, value) VALUES ('2016-12-02 23:50:00', 25.62359351633416); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:00:00', 25.503930477548035);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:10:00', 25.778348269011715); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:20:00', 25.61358296581122);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:30:00', 25.407082954413692); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:40:00', 25.59356132139061);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 00:50:00', 25.454359469668102); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:00:00', 25.600497611200172);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:10:00', 25.089287346535247); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:20:00', 25.406186385021478);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:30:00', 25.902770292173233); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:40:00', 25.430122083454933);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 01:50:00', 25.343677578353137); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:00:00', 25.827782785562448);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:10:00', 25.29812304633534); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:20:00', 25.449369799821127);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:30:00', 25.50067665405068); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:40:00', 25.9979864662843);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 02:50:00', 25.694267901014552); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:00:00', 25.483896119366477);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:10:00', 25.87093319303245); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:20:00', 25.45105002024843);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:30:00', 25.946018147464862); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:40:00', 25.15067665067811);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 03:50:00', 25.061446129380773); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:00:00', 25.421278070751292);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:10:00', 25.553324122715736); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:20:00', 25.193917339628353);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:30:00', 25.608770265474906); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:40:00', 25.005467597434503);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 04:50:00', 25.32181534608147); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:00:00', 25.603882488546358);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:10:00', 25.22053316373271); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:20:00', 25.68500870834277);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:30:00', 25.91769766933929); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:40:00', 25.849312452201982);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 05:50:00', 25.990145537303054); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:00:00', 25.937698567944693);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:10:00', 25.42855321297195); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:20:00', 25.178883393968068);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:30:00', 25.69575988830021); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:40:00', 25.172342442407984);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 06:50:00', 25.080605518088852); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:00:00', 25.78950316042253);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:10:00', 25.88138405813117); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:20:00', 25.353321832682024);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:30:00', 25.486967713164372); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:40:00', 25.241081032841397);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 07:50:00', 25.49196275290994); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:00:00', 25.45332850185998);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:10:00', 25.819180142996192); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:20:00', 25.41239031904037);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:30:00', 25.24367749100779); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:40:00', 30.27412775449895);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 08:50:00', 30.188777700355352); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:00:00', 30.446072701352517);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:10:00', 30.459609433778173); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:20:00', 30.257042607870012);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:30:00', 30.165281274620337); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:40:00', 30.226078491144943);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 09:50:00', 30.4460845208148); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:00:00', 30.270296036363806);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:10:00', 30.641048708539444); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:20:00', 30.14382111729586);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:30:00', 30.820109011539174); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:40:00', 30.84493226524782);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 10:50:00', 30.328083422720017); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:00:00', 30.60516210513217);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:10:00', 30.408059017291606); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:20:00', 30.34669916511969);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:30:00', 30.211917603793683); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:40:00', 30.366852988681565);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 11:50:00', 30.079332173258585); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:00:00', 30.163940918962364);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:10:00', 30.441891046089662); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:20:00', 30.097050080762052);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:30:00', 30.781660819725904); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:40:00', 30.556900724158172);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 12:50:00', 30.854915416016965); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:00:00', 30.76581319808981);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:10:00', 30.09515965534585); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:20:00', 30.85011299706575);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:30:00', 30.06167508067504); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:40:00', 30.613744288084394);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 13:50:00', 30.031460356397986); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:00:00', 30.576925005207194);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:10:00', 30.506954075147476); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:20:00', 30.583027116313506);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:30:00', 30.98952347335216); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:40:00', 30.537960928829108);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 14:50:00', 30.55906448897572); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:00:00', 30.938395927221197);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:10:00', 30.19125358459283); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:20:00', 30.89218489014141);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:30:00', 30.269881828803516); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:40:00', 30.62453863903956);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 15:50:00', 30.47049204000605); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:00:00', 30.929074483161077);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:10:00', 30.497084271977826); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:20:00', 30.23546928343591);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:30:00', 30.05088868062539); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:40:00', 30.479104535173093);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 16:50:00', 30.991924456036607); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:00:00', 30.1321253497226);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:10:00', 30.361037789125465); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:20:00', 30.567939373984228);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:30:00', 30.60608225546851); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:40:00', 30.93910000072389);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 17:50:00', 30.18435764271089); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:00:00', 30.084093443062766);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:10:00', 30.133540232917728); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:20:00', 30.415224360597637);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:30:00', 30.970271904292225); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:40:00', 30.392442033725175);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 18:50:00', 30.717345037830334); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:00:00', 30.29287254595604);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:10:00', 30.68001525671801); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:20:00', 30.323324484703765);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:30:00', 30.712905929174017); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:40:00', 30.237340925912207);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 19:50:00', 30.08066848279619); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:00:00', 30.90308092673425);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:10:00', 30.99790892366655); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:20:00', 30.75279348191689);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:30:00', 30.25620656041354); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:40:00', 30.340489058209528);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 20:50:00', 30.897623537636733); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:00:00', 30.37151989040234);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:10:00', 30.61349292258621); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:20:00', 30.816884181080642);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:30:00', 30.504656086416432); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:40:00', 30.87450944999693);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 21:50:00', 30.204642524976364); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:00:00', 30.767957826011852);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:10:00', 30.06720747943294); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:20:00', 30.94352337770597);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:30:00', 30.833975363783203); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:40:00', 30.001082718405442);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 22:50:00', 30.62722738811884); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:00:00', 30.419754881059927);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:10:00', 30.307988116789986); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:20:00', 30.040255900195287);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:30:00', 30.052270697149037); +INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:40:00', 30.75465363097227);INSERT INTO ts_data (date, value) VALUES ('2016-12-03 23:50:00', 30.25257093187005); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:00:00', 30.282273080528512);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:10:00', 30.427327140213684); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:20:00', 30.319335182679588);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:30:00', 30.37032099187025); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:40:00', 30.988634373631932);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 00:50:00', 30.828145873605614); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:00:00', 30.747086577208773);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:10:00', 30.267317734395885); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:20:00', 35.87736653308371);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:30:00', 35.213688135510274); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:40:00', 35.624399603171256);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 01:50:00', 35.54411296571213); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:00:00', 35.85002241567013);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:10:00', 35.18600923481531); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:20:00', 35.40299410979962);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:30:00', 35.09347255712887); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:40:00', 35.483986843607994);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 02:50:00', 35.138503824705126); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:00:00', 35.35293215080399);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:10:00', 35.60403409350849); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:20:00', 35.889313409443915);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:30:00', 35.376640675013235); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:40:00', 35.37812938577163);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 03:50:00', 35.340828488321115); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:00:00', 35.164237384320586);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:10:00', 35.372113230366274); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:20:00', 35.172726086784465);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:30:00', 35.4062303396216); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:40:00', 35.50774999212543);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 04:50:00', 35.69456814670008); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:00:00', 35.310291115843164);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:10:00', 35.769803853572924); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:20:00', 35.85937128671548);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:30:00', 35.450095945544774); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:40:00', 35.795843158352746);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 05:50:00', 35.3576696494664); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:00:00', 35.03003206031267);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:10:00', 35.79711747731257); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:20:00', 35.72215106257537);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:30:00', 35.87982201400974); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:40:00', 35.979599502978346);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 06:50:00', 35.240513882819045); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:00:00', 35.5556645216999);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:10:00', 35.67801636739599); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:20:00', 35.358328331146204);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:30:00', 35.890864610931736); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:40:00', 35.26681211624377);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 07:50:00', 35.879822712986986); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:00:00', 35.423068994010364);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:10:00', 35.3031232166475); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:20:00', 35.328828251473276);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:30:00', 35.687884043013135); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:40:00', 35.76018466466917);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 08:50:00', 35.80132183530506); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:00:00', 35.91475743887499);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:10:00', 35.672540184922454); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:20:00', 35.66439431430592);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:30:00', 35.28120978649861); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:40:00', 35.277339377103594);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 09:50:00', 35.01572217075871); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:00:00', 35.81763565574464);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:10:00', 35.60486763659024); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:20:00', 35.16818650113106);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:30:00', 35.86560072456925); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:40:00', 35.52020214456461);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 10:50:00', 35.23640805626655); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:00:00', 35.864600142011156);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:10:00', 35.89457994740151); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:20:00', 35.10484680070225);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:30:00', 35.60555515907971); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:40:00', 35.69632849630121);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 11:50:00', 35.58037240649336); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:00:00', 35.55190868293294);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:10:00', 35.31340455136142); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:20:00', 35.94962996624675);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:30:00', 35.30671500622937); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:40:00', 35.818865694354976);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 12:50:00', 35.621550979572355); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:00:00', 35.592736904726955);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:10:00', 35.82557283163196); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:20:00', 35.755573670923816);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:30:00', 35.169409433016675); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:40:00', 35.13349100077494);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 13:50:00', 35.2909332102636); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:00:00', 35.660308984561546);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:10:00', 35.81682683853163); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:20:00', 35.946497826369246);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:30:00', 35.303284063349686); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:40:00', 35.21681100014162);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 14:50:00', 35.60441856508883); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:00:00', 35.788542557562636);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:10:00', 35.261086449108404); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:20:00', 35.84207395018552);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:30:00', 35.13737421901026); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:40:00', 35.58797977996542);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 15:50:00', 35.82303019515759); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:00:00', 35.26929518692598);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:10:00', 35.27228631099635); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:20:00', 35.123823725129014);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:30:00', 35.11246238242373); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:40:00', 35.590660113589934);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 16:50:00', 35.41334554952506); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:00:00', 35.651952172286485);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:10:00', 35.106181340216715); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:20:00', 35.4567457663772);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:30:00', 35.8275466244328); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:40:00', 35.184825707890894);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 17:50:00', 35.47984459134186); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:00:00', 40.201337084523104);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:10:00', 40.84734870827102); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:20:00', 40.73705606414366);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:30:00', 40.73252660487228); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:40:00', 40.172410245348146);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 18:50:00', 40.048828795606454); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:00:00', 40.983784941808956);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:10:00', 40.06515187639006); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:20:00', 40.43276045789533);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:30:00', 40.20081847790017); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:40:00', 40.19821213906828);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 19:50:00', 40.94956311116983); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:00:00', 40.29817086437369);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:10:00', 40.431938065237006); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:20:00', 40.36603231123525);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:30:00', 40.132146902458615); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:40:00', 40.588114927354994);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 20:50:00', 40.60736948424093); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:00:00', 40.03099858528583);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:10:00', 40.938607980030326); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:20:00', 40.03294207429186);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:30:00', 40.764112050680716); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:40:00', 40.672025874759676);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 21:50:00', 40.30584769034546); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:00:00', 40.94998335104924);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:10:00', 40.77396252505749); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:20:00', 40.91530244740165);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:30:00', 40.14370527124542); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:40:00', 40.590636278757756);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 22:50:00', 40.24156378633403); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:00:00', 40.073198252283404);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:10:00', 40.32188296069385); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:20:00', 40.05834932525424);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:30:00', 40.98009611347602); +INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:40:00', 40.807081030095226);INSERT INTO ts_data (date, value) VALUES ('2016-12-04 23:50:00', 40.416308596525546); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:00:00', 40.195406604095496);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:10:00', 40.012208317863994); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:20:00', 40.997351313310965);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:30:00', 40.001139848708185); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:40:00', 40.11660570442965);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 00:50:00', 40.28369726867717); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:00:00', 40.566731939172016);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:10:00', 40.44968007271433); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:20:00', 40.03636727684547);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:30:00', 40.70737509962399); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:40:00', 40.13617661180935);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 01:50:00', 40.03591179587625); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:00:00', 40.79305684132999);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:10:00', 40.73273677300458); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:20:00', 40.26359489760031);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:30:00', 40.22100136997964); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:40:00', 40.02540702968399);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 02:50:00', 40.66516481797407); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:00:00', 40.809816864317504);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:10:00', 40.42502142809004); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:20:00', 40.594666767762455);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:30:00', 40.13449322396993); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:40:00', 40.40481232479665);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 03:50:00', 40.57203754327355); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:00:00', 40.59849946478817);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:10:00', 40.71961441011085); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:20:00', 40.16141923244062);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:30:00', 40.6512274207581); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:40:00', 40.507970240205545);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 04:50:00', 40.276511151846144); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:00:00', 40.798481342643385);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:10:00', 40.73265336679539); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:20:00', 40.285347037034136);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:30:00', 40.31497623962321); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:40:00', 40.4246202741558);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 05:50:00', 40.07422963599871); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:00:00', 40.12958348874789);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:10:00', 40.86818067987339); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:20:00', 40.642069489007156);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:30:00', 40.596095805955095); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:40:00', 40.52245777203022);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 06:50:00', 40.8366015203503); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:00:00', 40.512374857344334);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:10:00', 40.67906786720226); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:20:00', 40.76433461725766);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:30:00', 40.044167537846505); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:40:00', 40.41353814037301);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 07:50:00', 40.201682436018416); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:00:00', 40.623608235696494);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:10:00', 40.18297876833851); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:20:00', 40.688843141973095);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:30:00', 40.5891966023212); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:40:00', 40.522140381402785);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 08:50:00', 40.92296777389312); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:00:00', 40.24515676391933);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:10:00', 40.655731796548345); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:20:00', 40.92931397812652);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:30:00', 40.68752072547414); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:40:00', 40.32399036791148);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 09:50:00', 40.25529336316137); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:00:00', 40.10505267289531);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:10:00', 40.5226174091793); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:20:00', 40.86920004184902);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:30:00', 40.1906360993813); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:40:00', 45.85960967277176);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 10:50:00', 45.85471011877021); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:00:00', 45.1989877709722);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:10:00', 45.3504578840903); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:20:00', 45.91299356491165);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:30:00', 45.01371428303895); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:40:00', 45.728166750892846);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 11:50:00', 45.03541560229253); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:00:00', 45.5986239641956);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:10:00', 45.81923918989048); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:20:00', 45.618445773675745);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:30:00', 45.30979336518993); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:40:00', 45.15934246555148);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 12:50:00', 45.18658038758665); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:00:00', 45.742299566197865);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:10:00', 45.47496558470984); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:20:00', 45.17284107060644);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:30:00', 45.325728528577365); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:40:00', 45.75261959120912);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 13:50:00', 45.603756566101396); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:00:00', 45.91894555894857);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:10:00', 45.242366534994744); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:20:00', 45.57919077554701);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:30:00', 45.33679909615693); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:40:00', 45.20538737277241);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 14:50:00', 45.22278819604679); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:00:00', 45.23833000477602);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:10:00', 45.16959916289375); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:20:00', 45.45263959451466);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:30:00', 45.97908659091804); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:40:00', 45.140983413820756);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 15:50:00', 45.86277400548062); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:00:00', 45.97219831605039);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:10:00', 45.04580984090806); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:20:00', 45.59774031848686);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:30:00', 45.63530423956679); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:40:00', 45.417438997189045);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 16:50:00', 45.028671692966654); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:00:00', 45.40639490081234);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:10:00', 45.16304868205777); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:20:00', 45.049404899156826);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:30:00', 45.23123606201219); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:40:00', 45.487575568542034);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 17:50:00', 45.812505373419285); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:00:00', 45.1998713106634);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:10:00', 45.91470362508454); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:20:00', 45.88477858938986);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:30:00', 45.70057539782315); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:40:00', 45.92421826179503);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 18:50:00', 45.74893122420723); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:00:00', 45.7943930999712);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:10:00', 45.82325298584645); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:20:00', 45.66569344248397);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:30:00', 45.67425658687078); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:40:00', 45.22746463473383);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 19:50:00', 45.17154717754609); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:00:00', 45.10854903438189);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:10:00', 45.43473479850994); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:20:00', 45.86222726916496);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:30:00', 45.05271695541182); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:40:00', 45.21940679245271);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 20:50:00', 45.63379343662304); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:00:00', 45.83825662839634);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:10:00', 45.5920962016897); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:20:00', 45.29665460968132);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:30:00', 45.55980435003492); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:40:00', 45.29945324344045);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 21:50:00', 45.64509704799861); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:00:00', 45.93516281288697);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:10:00', 45.37087470863014); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:20:00', 45.38291167583321);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:30:00', 45.026662485818065); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:40:00', 45.948698660100106);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 22:50:00', 45.89348000261725); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:00:00', 45.27649342522346);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:10:00', 45.85725180589345); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:20:00', 45.02114997882529);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:30:00', 45.06405539268047); +INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:40:00', 45.80949949880207);INSERT INTO ts_data (date, value) VALUES ('2016-12-05 23:50:00', 45.495786881922356); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:00:00', 45.720225241407);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:10:00', 45.933869209111215); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:20:00', 45.48615887788706);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:30:00', 45.173105831338226); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:40:00', 45.32040103611929);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 00:50:00', 45.217329152085284); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:00:00', 45.70123831210879);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:10:00', 45.86375901353405); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:20:00', 45.51706924224786);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:30:00', 45.64083981618546); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:40:00', 45.846289361621196);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 01:50:00', 45.01135727097061); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:00:00', 45.8523614882762);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:10:00', 45.49417126000201); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:20:00', 45.075978494534034);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:30:00', 45.39386395911032); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:40:00', 45.28440126143178);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 02:50:00', 45.74746698766563); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:00:00', 45.102120344454384);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:10:00', 45.48826104948018); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:20:00', 50.04130376664194);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:30:00', 50.727901163386775); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:40:00', 50.87199601112955);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 03:50:00', 50.883147649282215); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:00:00', 50.412389559969775);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:10:00', 50.34857424700034); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:20:00', 50.215813927173606);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:30:00', 50.018098469361924); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:40:00', 50.73425346749341);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 04:50:00', 50.002531690603256); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:00:00', 50.11721308628489);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:10:00', 50.009792988413075); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:20:00', 50.32497669421603);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:30:00', 50.37683573640825); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:40:00', 50.051474942952034);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 05:50:00', 50.07289474404618); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:00:00', 50.391262596452464);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:10:00', 50.615218842210474); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:20:00', 50.57679177530203);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:30:00', 50.169333861882826); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:40:00', 50.467064754194354);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 06:50:00', 50.970678718984395); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:00:00', 50.02344103655669);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:10:00', 50.7407045279223); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:20:00', 50.629181974188796);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:30:00', 50.31194882276552); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:40:00', 50.81796755115351);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 07:50:00', 50.6852589724782); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:00:00', 50.10591157286973);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:10:00', 50.74949084724007); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:20:00', 50.86766143128229);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:30:00', 50.52841254042059); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:40:00', 50.21186794420065);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 08:50:00', 50.23745113566411); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:00:00', 50.80771016250989);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:10:00', 50.48986958238247); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:20:00', 50.28259222979104);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:30:00', 50.161565650554884); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:40:00', 50.51860921354403);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 09:50:00', 50.96407846353788); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:00:00', 50.323549622760275);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:10:00', 50.916330159524314); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:20:00', 50.85587878745068);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:30:00', 50.65454538449593); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:40:00', 50.675497805003396);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 10:50:00', 50.408827223672006); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:00:00', 50.89433758139433);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:10:00', 50.03073008577499); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:20:00', 50.32301210284869);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:30:00', 50.116087557817785); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:40:00', 50.45002574484946);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 11:50:00', 50.48432457719473); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:00:00', 50.59331195216032);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:10:00', 50.25303251796372); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:20:00', 50.3226011973296);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:30:00', 50.8550816464231); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:40:00', 50.99693944847967);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 12:50:00', 50.53003507084809); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:00:00', 50.44995606064558);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:10:00', 50.962455062018996); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:20:00', 50.04440554788321);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:30:00', 50.006627699043655); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:40:00', 50.5233631580475);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 13:50:00', 50.14425594578732); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:00:00', 50.62328506291849);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:10:00', 50.20417859692058); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:20:00', 50.33791801275508);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:30:00', 50.941081823467684); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:40:00', 50.27692920354136);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 14:50:00', 50.02908356520346); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:00:00', 50.843480719728326);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:10:00', 50.99652467140378); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:20:00', 50.83933192071146);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:30:00', 50.719743522754925); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:40:00', 50.8185853508712);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 15:50:00', 50.728112920211274); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:00:00', 50.93564338864228);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:10:00', 50.93162421598968); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:20:00', 50.682675687135884);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:30:00', 50.3653949442103); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:40:00', 50.693278186874196);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 16:50:00', 50.19673957513225); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:00:00', 50.84078682444565);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:10:00', 50.49072691853029); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:20:00', 50.71455809050517);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:30:00', 50.40283576568372); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:40:00', 50.09553898931378);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 17:50:00', 50.5771581130709); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:00:00', 50.8512641265803);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:10:00', 50.99025618868679); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:20:00', 50.54990302126502);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:30:00', 50.02189647716469); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:40:00', 50.5860172158262);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 18:50:00', 50.77702741926138); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:00:00', 50.69985640465449);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:10:00', 50.44780132561315); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:20:00', 50.44646297667806);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:30:00', 50.790495880398595); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:40:00', 50.66350390527233);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 19:50:00', 50.67518770201878); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:00:00', 55.86461671925514);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:10:00', 55.78305592759162); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:20:00', 55.11197041167928);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:30:00', 55.515116152767966); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:40:00', 55.46584376774793);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 20:50:00', 55.913224447583026); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:00:00', 55.88845127660456);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:10:00', 55.52716976049961); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:20:00', 55.76876652590235);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:30:00', 55.025615294620316); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:40:00', 55.577111109691664);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 21:50:00', 55.75536154797752); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:00:00', 55.713118360100154);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:10:00', 55.028975567513676); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:20:00', 55.92508590202515);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:30:00', 55.41531042821187); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:40:00', 55.22101104884555);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 22:50:00', 55.88218816525603); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:00:00', 55.71673883942393);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:10:00', 55.5404089312301); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:20:00', 55.86064551474242);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:30:00', 55.58431548296966); +INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:40:00', 55.78043236004886);INSERT INTO ts_data (date, value) VALUES ('2016-12-06 23:50:00', 55.14608087048665); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:00:00', 55.754244136434096);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:10:00', 55.364317872580244); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:20:00', 55.03948319958774);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:30:00', 55.85517637116233); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:40:00', 55.3587811811092);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 00:50:00', 55.826617148199254); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:00:00', 55.271131376795395);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:10:00', 55.23127750798324); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:20:00', 55.68116746960627);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:30:00', 55.03574324586153); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:40:00', 55.413332995916996);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 01:50:00', 55.23178134439042); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:00:00', 55.96892214895804);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:10:00', 55.11684222383856); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:20:00', 55.83445791300146);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:30:00', 55.57400857835226); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:40:00', 55.93485961438986);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 02:50:00', 55.429841228530314); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:00:00', 55.08035236743577);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:10:00', 55.41989113765008); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:20:00', 55.805544979426);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:30:00', 55.95946278699081); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:40:00', 55.54599750274753);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 03:50:00', 55.53445712721546); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:00:00', 55.16151681479046);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:10:00', 55.8920383082977); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:20:00', 55.72802993943361);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:30:00', 55.25451039355101); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:40:00', 55.99165299725528);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 04:50:00', 55.141520822030465); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:00:00', 55.817497050028884);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:10:00', 55.5550355233503); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:20:00', 55.05093545638608);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:30:00', 55.298948039581134); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:40:00', 55.96034981420127);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 05:50:00', 55.25624307611871); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:00:00', 55.18341244297447);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:10:00', 55.65755779828505); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:20:00', 55.62840711299797);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:30:00', 55.44172257257327); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:40:00', 55.94064309141799);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 06:50:00', 55.0708638460396); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:00:00', 55.014370631496924);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:10:00', 55.561869714864166); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:20:00', 55.997882720027505);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:30:00', 55.53117501273829); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:40:00', 55.12045803801976);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 07:50:00', 55.5305140970909); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:00:00', 55.18576884352693);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:10:00', 55.368122884800975); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:20:00', 55.46407661104355);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:30:00', 55.6038584617503); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:40:00', 55.10853132281619);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 08:50:00', 55.13139267312611); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:00:00', 55.69495905228585);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:10:00', 55.42283217515164); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:20:00', 55.60362109918761);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:30:00', 55.94376708230326); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:40:00', 55.7073472033042);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 09:50:00', 55.34929907927907); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:00:00', 55.14283314247304);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:10:00', 55.736943441535175); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:20:00', 55.174698766698505);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:30:00', 55.487672178112994); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:40:00', 55.10799408252901);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 10:50:00', 55.315593534425204); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:00:00', 55.59579859772211);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:10:00', 55.45114958941346); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:20:00', 55.66004661399281);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:30:00', 55.71685388314182); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:40:00', 55.068012354321745);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 11:50:00', 55.654963998005535); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:00:00', 55.17723467236714);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:10:00', 55.75479506184008); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:20:00', 55.15778134521637);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:30:00', 55.795424639230454); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:40:00', 60.42824126662819);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 12:50:00', 60.5757671418157); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:00:00', 60.1404810626188);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:10:00', 60.509704911130534); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:20:00', 60.646924356417465);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:30:00', 60.006959892353706); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:40:00', 60.77657049248974);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 13:50:00', 60.46017123295519); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:00:00', 60.48401439712486);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:10:00', 60.66858749935577); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:20:00', 60.96510334799966);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:30:00', 60.1187661935626); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:40:00', 60.89901268733274);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 14:50:00', 60.7100584149522); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:00:00', 60.343916158147145);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:10:00', 60.777117979126956); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:20:00', 60.73457150634369);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:30:00', 60.5304253179845); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:40:00', 60.72169834087641);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 15:50:00', 60.965641344091345); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:00:00', 60.217044491210146);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:10:00', 60.06542975986038); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:20:00', 60.32427943162242);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:30:00', 60.906465781861016); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:40:00', 60.461791464153684);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 16:50:00', 60.90000624945958); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:00:00', 60.24177596799514);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:10:00', 60.14024047215926); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:20:00', 60.98749731612882);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:30:00', 60.08592227499234); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:40:00', 60.35343869161658);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 17:50:00', 60.13885248255164); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:00:00', 60.35450042684454);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:10:00', 60.03211618097429); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:20:00', 60.273187737561074);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:30:00', 60.305472400797136); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:40:00', 60.59858406547435);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 18:50:00', 60.716684350836275); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:00:00', 60.40073723472085);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:10:00', 60.332361240318995); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:20:00', 60.19975296613247);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:30:00', 60.94185056034696); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:40:00', 60.27306202553091);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 19:50:00', 60.419961754944936); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:00:00', 60.68711857415286);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:10:00', 60.308048563696616); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:20:00', 60.43288153503136);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:30:00', 60.22240610394697); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:40:00', 60.87432112575496);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 20:50:00', 60.42657024373679); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:00:00', 60.437605292397);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:10:00', 60.09059057790051); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:20:00', 60.280926312745045);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:30:00', 60.228427584432715); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:40:00', 60.95304592243521);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 21:50:00', 60.54682796239075); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:00:00', 60.46454523913678);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:10:00', 60.51809831891373); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:20:00', 60.076953460188854);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:30:00', 60.499199498516006); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:40:00', 60.452895701794034);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 22:50:00', 60.20413704551565); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:00:00', 60.07591539489422);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:10:00', 60.63127640945167); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:20:00', 60.38481450413325);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:30:00', 60.70560320729101); +INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:40:00', 60.91809569688567);INSERT INTO ts_data (date, value) VALUES ('2016-12-07 23:50:00', 60.383938836598844); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:00:00', 60.67010478271658);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:10:00', 60.42291672933783); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:20:00', 60.31070562512965);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:30:00', 60.25651888575585); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:40:00', 60.36355352853065);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 00:50:00', 60.623416188823); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:00:00', 60.29259214493167);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:10:00', 60.49130446276643); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:20:00', 60.505422467667636);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:30:00', 60.582644388830175); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:40:00', 60.71386345313682);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 01:50:00', 60.931482176620534); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:00:00', 60.38112402586485);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:10:00', 60.99702588462689); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:20:00', 60.03308034897499);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:30:00', 60.05761908007053); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:40:00', 60.1214402114995);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 02:50:00', 60.88269160882629); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:00:00', 60.80626185025086);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:10:00', 60.774934578921076); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:20:00', 60.823458712724495);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:30:00', 60.52168862595384); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:40:00', 60.73565510163873);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 03:50:00', 60.60350497776844); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:00:00', 60.68478473293711);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:10:00', 60.346168541196306); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:20:00', 60.9149565352909);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:30:00', 60.28945564131935); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:40:00', 60.26923362436257);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 04:50:00', 60.01858550817403); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:00:00', 60.17470875147571);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:10:00', 60.604185712804565); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:20:00', 65.75717243160803);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:30:00', 65.07422746865474); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:40:00', 65.46395803181234);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 05:50:00', 65.32164179463363); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:00:00', 65.97344273308903);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:10:00', 65.16434989087553); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:20:00', 65.37884619243778);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:30:00', 65.97414561220593); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:40:00', 65.573338910014);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 06:50:00', 65.32982095211847); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:00:00', 65.11231290469598);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:10:00', 65.92361485867106); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:20:00', 65.37560800677892);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:30:00', 65.71195976194832); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:40:00', 65.262703659211);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 07:50:00', 65.15018180869143); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:00:00', 65.65495183113529);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:10:00', 65.05729917148591); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:20:00', 65.36401053057132);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:30:00', 65.63085936740404); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:40:00', 65.98756212918181);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 08:50:00', 65.75438329668366); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:00:00', 65.73123308297751);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:10:00', 65.3516908718983); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:20:00', 65.53636095470429);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:30:00', 65.08651350564215); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:40:00', 65.21751895177366);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 09:50:00', 65.3719973788212); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:00:00', 65.6405198875497);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:10:00', 65.13481851433889); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:20:00', 65.90554867795352);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:30:00', 65.08230521755553); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:40:00', 65.23338715435199);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 10:50:00', 65.81721113896879); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:00:00', 65.92490658172284);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:10:00', 65.8295466301566); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:20:00', 65.2709442164126);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:30:00', 65.67910457842821); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:40:00', 65.60570866531853);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 11:50:00', 65.18723696283372); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:00:00', 65.31752146342961);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:10:00', 65.69913903281); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:20:00', 65.02145958030293);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:30:00', 65.61372709361585); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:40:00', 65.44635354489421);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 12:50:00', 65.62965622782808); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:00:00', 65.35589597584978);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:10:00', 65.41784713643604); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:20:00', 65.18958358892179);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:30:00', 65.36613160951111); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:40:00', 65.50508385443378);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 13:50:00', 65.1803100219306); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:00:00', 65.05908329669485);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:10:00', 65.18186863371146); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:20:00', 65.61694445144289);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:30:00', 65.79260171668231); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:40:00', 65.2952654373761);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 14:50:00', 65.45914597305739); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:00:00', 65.01153576684446);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:10:00', 65.74243000191585); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:20:00', 65.81355219156411);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:30:00', 65.4047858511711); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:40:00', 65.27522667732593);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 15:50:00', 65.34709539941646); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:00:00', 65.8323582407959);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:10:00', 65.28052350397084); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:20:00', 65.21648824569458);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:30:00', 65.74454725415396); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:40:00', 65.94040871614496);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 16:50:00', 65.35980720632816); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:00:00', 65.01297183319355);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:10:00', 65.03731826900781); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:20:00', 65.96657288125054);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:30:00', 65.11871187478256); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:40:00', 65.60985795474032);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 17:50:00', 65.48919459176929); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:00:00', 65.908713534529);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:10:00', 65.32591704286483); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:20:00', 65.93247438584325);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:30:00', 65.25551421491285); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:40:00', 65.73367349897329);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 18:50:00', 65.07091664126513); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:00:00', 65.07512162349447);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:10:00', 65.7151484417674); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:20:00', 65.43895169098508);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:30:00', 65.1805144957315); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:40:00', 65.70261769381938);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 19:50:00', 65.56150506790509); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:00:00', 65.23764142715255);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:10:00', 65.96850090859809); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:20:00', 65.60991514907558);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:30:00', 65.0137033690365); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:40:00', 65.55642960976178);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 20:50:00', 65.4291480698173); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:00:00', 65.7948850825054);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:10:00', 65.68857036904905); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:20:00', 65.88982844642518);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:30:00', 65.6797902956207); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:40:00', 65.04828322798917);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 21:50:00', 65.98121258744061); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:00:00', 70.84792375176262);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:10:00', 70.24714401393534); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:20:00', 70.82080353559094);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:30:00', 70.22493475440335); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:40:00', 70.83517576657364);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 22:50:00', 70.48220481199782); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:00:00', 70.50987238063006);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:10:00', 70.65887788836065); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:20:00', 70.15813700129239);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:30:00', 70.93666891618649); +INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:40:00', 70.09444659260299);INSERT INTO ts_data (date, value) VALUES ('2016-12-08 23:50:00', 70.44610246379736); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:00:00', 70.60755376156217);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:10:00', 70.62952517971817); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:20:00', 70.8341430461983);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:30:00', 70.54258873073861); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:40:00', 70.78584097729174);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 00:50:00', 70.56133790996552); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:00:00', 70.00926091252859);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:10:00', 70.84692164609287); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:20:00', 70.81560829042392);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:30:00', 70.4564813058073); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:40:00', 70.53922518502625);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 01:50:00', 70.29454246182891); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:00:00', 70.62694229701485);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:10:00', 70.77008506994437); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:20:00', 70.90039896257827);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:30:00', 70.20802685112797); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:40:00', 70.23043617175695);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 02:50:00', 70.93868928146462); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:00:00', 70.32214900023658);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:10:00', 70.98721417278793); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:20:00', 70.7073862675702);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:30:00', 70.07073125003258); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:40:00', 70.21898268089228);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 03:50:00', 70.0459327620224); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:00:00', 70.47883044279872);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:10:00', 70.0357550187124); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:20:00', 70.9029467289573);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:30:00', 70.19262260368063); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:40:00', 70.67181564949811);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 04:50:00', 70.54165230053579); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:00:00', 70.20576791812435);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:10:00', 70.97042118947293); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:20:00', 70.55995951405272);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:30:00', 70.59708865561609); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:40:00', 70.61280063207539);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 05:50:00', 70.11275312715213); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:00:00', 70.40198693106453);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:10:00', 70.52719613265567); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:20:00', 70.36503580697642);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:30:00', 70.35992503082046); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:40:00', 70.89632985223194);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 06:50:00', 70.31524736062875); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:00:00', 70.14663551460536);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:10:00', 70.629373723081); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:20:00', 70.45578605413314);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:30:00', 70.37049605560195); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:40:00', 70.33728568741186);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 07:50:00', 70.0374832297144); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:00:00', 70.46953234119924);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:10:00', 70.44757344459161); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:20:00', 70.61022791965419);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:30:00', 70.44489295012828); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:40:00', 70.91419584310704);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 08:50:00', 70.14092066876647); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:00:00', 70.49788868528776);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:10:00', 70.2505423822712); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:20:00', 70.82541110065833);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:30:00', 70.49822564109441); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:40:00', 70.23604429880052);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 09:50:00', 70.47028795047646); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:00:00', 70.52196524379626);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:10:00', 70.73173576741272); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:20:00', 70.59085505060818);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:30:00', 70.61416398517015); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:40:00', 70.35856993530311);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 10:50:00', 70.79628009722434); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:00:00', 70.39124910067879);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:10:00', 70.3169644539236); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:20:00', 70.66814559254944);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:30:00', 70.66727306779204); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:40:00', 70.59748311984364);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 11:50:00', 70.63137510919809); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:00:00', 70.15372199828549);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:10:00', 70.94534209063247); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:20:00', 70.11126223644388);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:30:00', 70.62673987071862); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:40:00', 70.55893655546298);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 12:50:00', 70.85913955652822); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:00:00', 70.44300398324273);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:10:00', 70.92669544436872); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:20:00', 70.68500041259618);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:30:00', 70.73821146363592); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:40:00', 70.09185238051528);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 13:50:00', 70.52442445866551); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:00:00', 70.76453775181325);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:10:00', 70.87844349736861); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:20:00', 70.10201299228585);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:30:00', 70.73573441334291); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:40:00', 75.73875085565189);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 14:50:00', 75.68924076910906); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:00:00', 75.40249603606367);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:10:00', 75.10192311249615); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:20:00', 75.23763558581453);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:30:00', 75.33174535792007); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:40:00', 75.13750709632737);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 15:50:00', 75.87624497874643); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:00:00', 75.51507412723397);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:10:00', 75.17984288750057); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:20:00', 75.57942666194583);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:30:00', 75.17559970453412); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:40:00', 75.57867691398481);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 16:50:00', 75.5768138086326); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:00:00', 75.18609446940691);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:10:00', 75.23104725966235); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:20:00', 75.0406505098033);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:30:00', 75.10486663190895); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:40:00', 75.91293660240508);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 17:50:00', 75.48879327914106); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:00:00', 75.87941307619892);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:10:00', 75.59084950346809); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:20:00', 75.1040794971801);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:30:00', 75.30164181144724); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:40:00', 75.77590817467764);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 18:50:00', 75.6942164667002); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:00:00', 75.94869729038744);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:10:00', 75.72456322147062); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:20:00', 75.04615245346291);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:30:00', 75.08142037966856); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:40:00', 75.82421514583359);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 19:50:00', 75.99785306189705); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:00:00', 75.68095826431698);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:10:00', 75.81478375577895); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:20:00', 75.43084888893065);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:30:00', 75.3865179272271); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:40:00', 75.85559764019484);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 20:50:00', 75.3179213371592); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:00:00', 75.99517990722951);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:10:00', 75.77330732423295); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:20:00', 75.41675859559655);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:30:00', 75.61660863738516); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:40:00', 75.29748058931841);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 21:50:00', 75.86044161935821); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:00:00', 75.68189971766895);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:10:00', 75.52662363332475); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:20:00', 75.99381551449784);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:30:00', 75.09372986941132); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:40:00', 75.76333867961407);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 22:50:00', 75.98746157570056); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:00:00', 75.59844485443448);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:10:00', 75.11850856645286); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:20:00', 75.85650339910504);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:30:00', 75.73934452601394); +INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:40:00', 75.70413465820144);INSERT INTO ts_data (date, value) VALUES ('2016-12-09 23:50:00', 75.73277653616444); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:00:00', 75.52897698618867);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:10:00', 75.06683383389577); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:20:00', 75.10826702568332);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:30:00', 75.92204167961121); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:40:00', 75.67334637926871);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 00:50:00', 75.0098275704047); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:00:00', 75.79034949378887);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:10:00', 75.25034943540432); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:20:00', 75.2418857167868);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:30:00', 75.55151921991957); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:40:00', 75.22659343226981);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 01:50:00', 75.6297760503289); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:00:00', 75.60625964704498);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:10:00', 75.92514383190209); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:20:00', 75.33584156667379);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:30:00', 75.45888705086357); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:40:00', 75.1171019402213);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 02:50:00', 75.34421798283691); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:00:00', 75.70906630644397);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:10:00', 75.13314786525794); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:20:00', 75.82994660760363);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:30:00', 75.05586028954062); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:40:00', 75.72739447197894);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 03:50:00', 75.27139953673046); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:00:00', 75.0373887193959);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:10:00', 75.58664575769784); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:20:00', 75.64542070383824);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:30:00', 75.93361623534472); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:40:00', 75.21727104704924);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 04:50:00', 75.39678815950646); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:00:00', 75.09570509331364);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:10:00', 75.97729622254738); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:20:00', 75.0855718432743);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:30:00', 75.97580472209289); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:40:00', 75.42239611691534);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 05:50:00', 75.65532024535044); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:00:00', 75.78582977910041);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:10:00', 75.49909674660347); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:20:00', 75.61170402506708);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:30:00', 75.30708138689313); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:40:00', 75.45874058634112);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 06:50:00', 75.67144979152366); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:00:00', 75.4767990754182);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:10:00', 75.03148874161155); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:20:00', 80.05177827557155);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:30:00', 80.55125618457271); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:40:00', 80.56327881372859);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 07:50:00', 80.62561917424894); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:00:00', 80.13358094872449);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:10:00', 80.57447479351923); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:20:00', 80.38762041089727);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:30:00', 80.22345147063619); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:40:00', 80.77662293919255);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 08:50:00', 80.06086260124913); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:00:00', 80.20648608787332);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:10:00', 80.24904308099917); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:20:00', 80.4470358500717);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:30:00', 80.10208054257717); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:40:00', 80.6948625345203);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 09:50:00', 80.20134542326788); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:00:00', 80.63746559248813);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:10:00', 80.09679600352392); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:20:00', 80.40261514796043);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:30:00', 80.35729868236676); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:40:00', 80.68933335226585);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 10:50:00', 80.62613946216034); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:00:00', 80.13081628909515);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:10:00', 80.83351790238845); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:20:00', 80.15002651009729);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:30:00', 80.6848971808227); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:40:00', 80.17155394414425);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 11:50:00', 80.01414372310468); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:00:00', 80.23153708385941);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:10:00', 80.1491872075093); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:20:00', 80.86763964224257);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:30:00', 80.27968860134462); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:40:00', 80.46846001239791);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 12:50:00', 80.7377358061797); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:00:00', 80.06297519616069);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:10:00', 80.47599450773214); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:20:00', 80.16928442772286);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:30:00', 80.1866406554142); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:40:00', 80.26220108849837);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 13:50:00', 80.80104515673855); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:00:00', 80.02502473593107);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:10:00', 80.65587704369372); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:20:00', 80.43734579422477);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:30:00', 80.4605157182123); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:40:00', 80.07320618895673);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 14:50:00', 80.83280728273998); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:00:00', 80.52997403844681);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:10:00', 80.47436185006217); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:20:00', 80.61573331558482);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:30:00', 80.53430778954069); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:40:00', 80.46207010848349);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 15:50:00', 80.10951598700812); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:00:00', 80.14204891643658);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:10:00', 80.37656701517018); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:20:00', 80.68481469910469);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:30:00', 80.78105279409999); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:40:00', 80.9563652599067);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 16:50:00', 80.72225111864762); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:00:00', 80.53065993656942);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:10:00', 80.93882701354313); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:20:00', 80.19051555622741);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:30:00', 80.05288737619873); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:40:00', 80.20363409046747);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 17:50:00', 80.00490482513175); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:00:00', 80.1987350455192);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:10:00', 80.72845003892684); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:20:00', 80.40568741749185);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:30:00', 80.8944897444389); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:40:00', 80.13821234591664);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 18:50:00', 80.80735853124939); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:00:00', 80.26587482706799);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:10:00', 80.28707259941027); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:20:00', 80.29978568164628);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:30:00', 80.19257750117515); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:40:00', 80.51644031350376);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 19:50:00', 80.20870394717966); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:00:00', 80.04208772645019);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:10:00', 80.74839009450095); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:20:00', 80.74270770219081);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:30:00', 80.27061805307784); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:40:00', 80.2121638737193);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 20:50:00', 80.44559311165203); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:00:00', 80.96841302878805);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:10:00', 80.62565250431959); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:20:00', 80.83997237788287);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:30:00', 80.83476992395882); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:40:00', 80.33982309292188);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 21:50:00', 80.6145748758383); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:00:00', 80.74159692216102);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:10:00', 80.23137179819544); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:20:00', 80.68413946439128);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:30:00', 80.60253970493986); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:40:00', 80.38803395921579);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 22:50:00', 80.41587051820059); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:00:00', 80.63414747366352);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:10:00', 80.17821559821772); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:20:00', 80.18485850974864);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:30:00', 80.96882915978566); +INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:40:00', 80.25494618536747);INSERT INTO ts_data (date, value) VALUES ('2016-12-10 23:50:00', 80.47145006805054); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:00:00', 85.07014084281235);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:10:00', 85.36053530006474); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:20:00', 85.72638672081307);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:30:00', 85.68543179856624); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:40:00', 85.06573949110016);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 00:50:00', 85.39619525156424); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:00:00', 85.34323892038135);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:10:00', 85.83612931876877); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:20:00', 85.38899679126838);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:30:00', 85.5845404775761); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:40:00', 85.49266890254829);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 01:50:00', 85.2020961319447); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:00:00', 85.70732433516498);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:10:00', 85.45316638950072); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:20:00', 85.60159799035198);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:30:00', 85.48513567233881); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:40:00', 85.72015763104838);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 02:50:00', 85.27140961968638); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:00:00', 85.06114371615334);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:10:00', 85.68284915594599); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:20:00', 85.86224158293518);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:30:00', 85.03062371816587); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:40:00', 85.73342399005355);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 03:50:00', 85.29459639238314); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:00:00', 85.4034171818368);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:10:00', 85.99601629298941); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:20:00', 85.58811032293947);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:30:00', 85.52973763102459); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:40:00', 85.56355686500426);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 04:50:00', 85.89108922003679); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:00:00', 85.9833866146756);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:10:00', 85.06044404268381); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:20:00', 85.77748280834021);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:30:00', 85.6837610301493); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:40:00', 85.15219723566939);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 05:50:00', 85.10435743280738); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:00:00', 85.37547815372379);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:10:00', 85.77748123606732); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:20:00', 85.54794351256511);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:30:00', 85.45151980009891); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:40:00', 85.39295800693596);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 06:50:00', 85.06802277138283); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:00:00', 85.55971158029409);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:10:00', 85.38164289249396); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:20:00', 85.10085197592385);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:30:00', 85.48725534100402); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:40:00', 85.52593256171464);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 07:50:00', 85.87156513298041); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:00:00', 85.07862802922249);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:10:00', 85.72336526868045); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:20:00', 85.82142217076826);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:30:00', 85.57039253696205); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:40:00', 85.12543146754433);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 08:50:00', 85.95003848846102); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:00:00', 85.93983330592356);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:10:00', 85.8894598818847); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:20:00', 85.55443573823122);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:30:00', 85.75699779977316); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:40:00', 85.66753770050688);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 09:50:00', 85.30928352007138); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:00:00', 85.66903513434067);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:10:00', 85.083443919299); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:20:00', 85.79800247593478);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:30:00', 85.01679441203906); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:40:00', 85.40328635972477);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 10:50:00', 85.36241590940696); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:00:00', 85.17806212490461);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:10:00', 85.78086871874406); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:20:00', 85.86432022718121);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:30:00', 85.14542697715467); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:40:00', 85.78580299913301);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 11:50:00', 85.79107965449309); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:00:00', 85.85390702180145);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:10:00', 85.83790102951062); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:20:00', 85.05869397189487);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:30:00', 85.42097335726912); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:40:00', 85.90493345730768);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 12:50:00', 85.57236725902774); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:00:00', 85.76671318462299);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:10:00', 85.9854775354553); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:20:00', 85.11491748659687);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:30:00', 85.65618507697398); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:40:00', 85.2976768974066);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 13:50:00', 85.790727502414); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:00:00', 85.31097224584342);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:10:00', 85.45443176058593); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:20:00', 85.4497890737567);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:30:00', 85.76434428610645); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:40:00', 85.95723995051942);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 14:50:00', 85.80410985528856); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:00:00', 85.42796805505756);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:10:00', 85.57439837703916); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:20:00', 85.55171629202331);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:30:00', 85.94574567876707); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:40:00', 85.84253228342556);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 15:50:00', 85.8231956362723); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:00:00', 85.56463113744121);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:10:00', 85.37280340640424); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:20:00', 85.08333657035288);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:30:00', 85.79176703962894); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:40:00', 90.9824625344212);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 16:50:00', 90.80670841671619); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:00:00', 90.95126333998554);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:10:00', 90.48883720548729); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:20:00', 90.2923260087112);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:30:00', 90.51452461979704); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:40:00', 90.69410598821828);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 17:50:00', 90.53662585457249); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:00:00', 90.28384547135903);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:10:00', 90.72836694448385); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:20:00', 90.79370407752636);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:30:00', 90.34427946065428); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:40:00', 90.91749139074152);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 18:50:00', 90.78235016294744); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:00:00', 90.81669047135937);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:10:00', 90.99144445409335); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:20:00', 90.05074397237844);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:30:00', 90.23587629854755); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:40:00', 90.42735397359165);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 19:50:00', 90.63333890728542); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:00:00', 90.45749785824628);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:10:00', 90.47511827899672); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:20:00', 90.90202603264449);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:30:00', 90.91886052209719); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:40:00', 90.82261955614763);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 20:50:00', 90.60251735636996); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:00:00', 90.60858268969284);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:10:00', 90.6901637526175); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:20:00', 90.79931191877024);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:30:00', 90.94564613566642); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:40:00', 90.52138817215868);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 21:50:00', 90.28452056335456); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:00:00', 90.0320292854833);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:10:00', 90.17065842730513); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:20:00', 90.56179378053595);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:30:00', 90.80209675698634); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:40:00', 90.51980296587465);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 22:50:00', 90.81073939488728); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:00:00', 90.21851907374479);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:10:00', 90.3350149456988); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:20:00', 90.93990581260924);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:30:00', 90.70897406397431); +INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:40:00', 90.00175599098391);INSERT INTO ts_data (date, value) VALUES ('2016-12-11 23:50:00', 90.25201589123915); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:00:00', 90.20684508994745);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:10:00', 90.30766125369664); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:20:00', 90.74912066619444);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:30:00', 90.46841740749424); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:40:00', 90.59370522649891);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 00:50:00', 90.2501217650573); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:00:00', 90.55961793814348);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:10:00', 90.27369721629333); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:20:00', 90.15913834803024);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:30:00', 90.24824580229117); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:40:00', 90.01527042304376);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 01:50:00', 90.78483738088785); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:00:00', 90.20246578626534);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:10:00', 90.42122852088525); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:20:00', 90.96079390424096);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:30:00', 90.39849247285359); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:40:00', 90.57641891985936);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 02:50:00', 90.11550147649305); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:00:00', 90.58461185786881);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:10:00', 90.02777344004706); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:20:00', 90.55556257005077);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:30:00', 90.83971627172713); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:40:00', 90.18166451119238);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 03:50:00', 90.625356986698); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:00:00', 90.59600460482297);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:10:00', 90.00920036129621); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:20:00', 90.26312938191263);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:30:00', 90.21126188660082); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:40:00', 90.27358248872356);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 04:50:00', 90.65075765804363); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:00:00', 90.56581960625195);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:10:00', 90.30095213443592); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:20:00', 90.5733945270204);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:30:00', 90.22087591168156); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:40:00', 90.70638255635788);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 05:50:00', 90.51904534190113); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:00:00', 90.93288107407612);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:10:00', 90.02738882847706); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:20:00', 90.94145397447491);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:30:00', 90.49313500741732); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:40:00', 90.99100815129243);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 06:50:00', 90.93467490108415); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:00:00', 90.1349313909047);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:10:00', 90.29207422425257); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:20:00', 90.71015752581968);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:30:00', 90.2276461972035); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:40:00', 90.24617387551187);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 07:50:00', 90.39927626803959); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:00:00', 90.30233537599464);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:10:00', 90.17441461736765); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:20:00', 90.78714162395201);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:30:00', 90.44790297372845); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:40:00', 90.7742848703115);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 08:50:00', 90.9967671844167); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:00:00', 90.29369758387823);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:10:00', 90.85977760577518); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:20:00', 95.70273919522026);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:30:00', 95.72655193466684); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:40:00', 95.68564644336149);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 09:50:00', 95.23140811939676); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:00:00', 95.41132878464948);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:10:00', 95.51410522097108); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:20:00', 95.12185543853779);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:30:00', 95.21711983477532); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:40:00', 95.08595493204972);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 10:50:00', 95.2690559077771); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:00:00', 95.8247084729692);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:10:00', 95.37897575959448); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:20:00', 95.9912168689898);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:30:00', 95.64427550907007); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:40:00', 95.53399676703508);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 11:50:00', 95.5746199904719); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:00:00', 95.9187378270099);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:10:00', 95.39134339392945); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:20:00', 95.7053585881445);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:30:00', 95.34373841660249); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:40:00', 95.78505865015656);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 12:50:00', 95.35279032777069); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:00:00', 95.36601508459272);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:10:00', 95.16843373559405); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:20:00', 95.14675336545551);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:30:00', 95.59360876887017); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:40:00', 95.05465673216129);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 13:50:00', 95.08114900094135); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:00:00', 95.20328901548919);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:10:00', 95.11776422317558); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:20:00', 95.09685773302864);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:30:00', 95.44575525609163); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:40:00', 95.76418151133755);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 14:50:00', 95.9433408206822); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:00:00', 95.43089972426745);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:10:00', 95.10083246997101); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:20:00', 95.31373144887291);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:30:00', 95.49404831529185); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:40:00', 95.12094902570072);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 15:50:00', 95.14421426311722); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:00:00', 95.03300409670959);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:10:00', 95.5977018635513); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:20:00', 95.20245201405494);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:30:00', 95.8042814980708); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:40:00', 95.80967702428563);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 16:50:00', 95.6310032550752); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:00:00', 95.78813452304236);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:10:00', 95.3710756559026); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:20:00', 95.84073139666731);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:30:00', 95.87565958347292); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:40:00', 95.27923779126085);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 17:50:00', 95.43791310863152); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:00:00', 95.37353522312424);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:10:00', 95.45483514345189); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:20:00', 95.27243362298515);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:30:00', 95.53472611364586); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:40:00', 95.9943171036817);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 18:50:00', 95.75241201361489); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:00:00', 95.662009154259);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:10:00', 95.8559140171521); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:20:00', 95.99341595419622);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:30:00', 95.96512948740204); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:40:00', 95.40885203737166);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 19:50:00', 95.40368419454084); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:00:00', 95.14626703644213);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:10:00', 95.16021693202427); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:20:00', 95.48791912324057);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:30:00', 95.86748213114836); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:40:00', 95.29693902248422);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 20:50:00', 95.74430025147372); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:00:00', 95.33353539080912);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:10:00', 95.24171867670813); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:20:00', 95.41393555195542);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:30:00', 95.24211730892031); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:40:00', 95.01001258896444);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 21:50:00', 95.22691557758905); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:00:00', 95.00702581134178);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:10:00', 95.24259265108412); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:20:00', 95.00263139795263);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:30:00', 95.1527736215333); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:40:00', 95.20922450679696);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 22:50:00', 95.63795697028615); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:00:00', 95.71452083922787);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:10:00', 95.7978817600075); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:20:00', 95.78574938389043);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:30:00', 95.3894335508658); +INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:40:00', 95.47191602261879);INSERT INTO ts_data (date, value) VALUES ('2016-12-12 23:50:00', 95.4161076668443); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:00:00', 95.28418844609357);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:10:00', 95.63241694113232); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:20:00', 95.93718662199717);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:30:00', 95.27693030242013); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:40:00', 95.94671082905306);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 00:50:00', 95.63096731898416); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:00:00', 95.70082329449929);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:10:00', 95.28902281965944); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:20:00', 95.3096473200545);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:30:00', 95.25709498620873); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:40:00', 95.76420220486904);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 01:50:00', 95.25611173480905); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:00:00', 100.06185651453708);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:10:00', 100.4580955110936); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:20:00', 100.02486306345648);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:30:00', 100.65935078621763); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:40:00', 100.80202938991845);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 02:50:00', 100.838535680746); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:00:00', 100.15437846704823);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:10:00', 100.00266102472185); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:20:00', 100.23413928641735);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:30:00', 100.9436158972604); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:40:00', 100.67054972371241);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 03:50:00', 100.21319010879192); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:00:00', 100.06564362855326);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:10:00', 100.25497290102619); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:20:00', 100.75353786140448);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:30:00', 100.2756056686153); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:40:00', 100.43988404887469);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 04:50:00', 100.24517273423756); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:00:00', 100.56119548395081);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:10:00', 100.9423620903701); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:20:00', 100.22213644715453);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:30:00', 100.07951390032152); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:40:00', 100.20968090775213);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 05:50:00', 100.78367715139623); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:00:00', 100.86252769643005);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:10:00', 100.78535813702729); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:20:00', 100.49325730700184);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:30:00', 100.03124276082373); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:40:00', 100.73817148850584);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 06:50:00', 100.54620495076551); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:00:00', 100.19007432703216);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:10:00', 100.69654359638416); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:20:00', 100.76761046634724);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:30:00', 100.56329351628469); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:40:00', 100.02109622349658);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 07:50:00', 100.41610251701651); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:00:00', 100.38134443340543);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:10:00', 100.52017412664446); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:20:00', 100.55825864280058);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:30:00', 100.93612685898168); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:40:00', 100.87176024259242);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 08:50:00', 100.09432075442085); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:00:00', 100.21522312629112);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:10:00', 100.0641885113497); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:20:00', 100.68295526141016);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:30:00', 100.64919442070853); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:40:00', 100.8513673974551);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 09:50:00', 100.57327938047924); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:00:00', 100.87063567639632);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:10:00', 100.26801292992012); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:20:00', 100.03971650334793);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:30:00', 100.06263305061634); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:40:00', 100.05275073481121);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 10:50:00', 100.72089880683666); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:00:00', 100.72130967371119);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:10:00', 100.46428694436828); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:20:00', 100.67646105198568);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:30:00', 100.65714458595468); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:40:00', 100.90429981851341);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 11:50:00', 100.62413165850428); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:00:00', 100.5684317347736);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:10:00', 100.48164986723944); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:20:00', 100.07108879370205);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:30:00', 100.78045834326109); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:40:00', 100.47189403300331);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 12:50:00', 100.0871521681507); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:00:00', 100.20977695685312);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:10:00', 100.58869246991378); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:20:00', 100.050495427379);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:30:00', 100.07176411567735); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:40:00', 100.66547261466386);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 13:50:00', 100.6255695225099); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:00:00', 100.52528545450022);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:10:00', 100.06630482528924); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:20:00', 100.54826862079224);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:30:00', 100.27743208948088); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:40:00', 100.59151187969195);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 14:50:00', 100.19357087141566); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:00:00', 100.2055761518432);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:10:00', 100.52294916116287); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:20:00', 100.22918892587309);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:30:00', 100.30122764892819); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:40:00', 100.46434409024792);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 15:50:00', 100.09147329074148); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:00:00', 100.673016061652);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:10:00', 100.05183192923042); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:20:00', 100.84874371063628);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:30:00', 100.97796658012604); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:40:00', 100.47187175409692);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 16:50:00', 100.04795930073355); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:00:00', 100.14495201018543);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:10:00', 100.7712073637323); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:20:00', 100.48376325881216);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:30:00', 100.02989656687107); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:40:00', 100.30850145078257);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 17:50:00', 100.79732770188521); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:00:00', 100.87191486590562);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:10:00', 100.99561345408628); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:20:00', 100.44482234718608);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:30:00', 100.54045807024269); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:40:00', 105.94962782322027);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 18:50:00', 105.98419551485628); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:00:00', 105.98450691349268);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:10:00', 105.30243117020771); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:20:00', 105.62442237163904);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:30:00', 105.19655848954201); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:40:00', 105.2678560621535);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 19:50:00', 105.52085156046782); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:00:00', 105.73739469337072);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:10:00', 105.43309356793623); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:20:00', 105.3890473200834);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:30:00', 105.30614100119176); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:40:00', 105.1108971300237);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 20:50:00', 105.39575816449894); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:00:00', 105.05690563154423);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:10:00', 105.0661670292193); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:20:00', 105.58161836708838);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:30:00', 105.04807950700233); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:40:00', 105.2419395813828);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 21:50:00', 105.71938038146294); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:00:00', 105.2768557272797);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:10:00', 105.32026065010504); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:20:00', 105.78523929626823);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:30:00', 105.276924615787); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:40:00', 105.17367190128779);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 22:50:00', 105.7840852531348); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:00:00', 105.12486918168095);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:10:00', 105.87540477798387); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:20:00', 105.71914313529292);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:30:00', 105.90351017547114); +INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:40:00', 105.92546541917544);INSERT INTO ts_data (date, value) VALUES ('2016-12-13 23:50:00', 105.95212937011351); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:00:00', 105.9877572102225);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:10:00', 105.00118159423285); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:20:00', 105.28564303203548);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:30:00', 105.1025275628095); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:40:00', 105.61896840936787);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 00:50:00', 105.18417419890764); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:00:00', 105.67061022532066);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:10:00', 105.47663142214905); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:20:00', 105.35190138496912);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:30:00', 105.90842581892744); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:40:00', 105.4222231037005);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 01:50:00', 105.76550949712649); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:00:00', 105.38575269295369);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:10:00', 105.87935680249542); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:20:00', 105.29297243716174);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:30:00', 105.31400347205394); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:40:00', 105.52715002910543);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 02:50:00', 105.78276795303265); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:00:00', 105.48506256143213);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:10:00', 105.72614615283683); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:20:00', 105.85079254979068);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:30:00', 105.09790941235916); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:40:00', 105.46933283027454);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 03:50:00', 105.70831466965087); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:00:00', 105.39008927983515);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:10:00', 105.1467233401007); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:20:00', 105.7448854928425);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:30:00', 105.40150169225193); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:40:00', 105.10797364238701);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 04:50:00', 105.4149393297828); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:00:00', 105.34898977354796);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:10:00', 105.71491489922285); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:20:00', 105.42653168477068);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:30:00', 105.54718526304302); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:40:00', 105.42521382345095);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 05:50:00', 105.67102658209608); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:00:00', 105.46038317508213);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:10:00', 105.79683626991262); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:20:00', 105.36926571740987);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:30:00', 105.28023685231967); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:40:00', 105.82166563832027);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 06:50:00', 105.29457431726064); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:00:00', 105.1349211067932);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:10:00', 105.6386492093234); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:20:00', 105.81027712583908);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:30:00', 105.59457413075762); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:40:00', 105.38090232975182);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 07:50:00', 105.89491581437031); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:00:00', 105.91463360016053);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:10:00', 105.92926587130607); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:20:00', 105.0471923488872);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:30:00', 105.69350925950778); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:40:00', 105.9500247412264);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 08:50:00', 105.52697621217106); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:00:00', 105.44494066914405);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:10:00', 105.91518954803983); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:20:00', 105.17932691918186);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:30:00', 105.0265545565388); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:40:00', 105.72834323917131);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 09:50:00', 105.3212046616362); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:00:00', 105.25101430465428);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:10:00', 105.51898599885591); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:20:00', 105.88603103416416);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:30:00', 105.6764609467606); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:40:00', 105.9437335262162);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 10:50:00', 105.56486129098856); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:00:00', 105.44014210790266);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:10:00', 105.27107612729998); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:20:00', 110.12766042119884);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:30:00', 110.46097328333843); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:40:00', 110.09650709715108);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 11:50:00', 110.530961479503); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:00:00', 110.29088215265074);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:10:00', 110.03711684433249); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:20:00', 110.55993032748897);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:30:00', 110.59914205427324); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:40:00', 110.49597522798216);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 12:50:00', 110.0851886686103); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:00:00', 110.74491457116008);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:10:00', 110.33007256532116); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:20:00', 110.96297477536591);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:30:00', 110.24294858126936); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:40:00', 110.21683982012392);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 13:50:00', 110.64872681334936); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:00:00', 110.33724372312577);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:10:00', 110.57120569484695); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:20:00', 110.84902105905329);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:30:00', 110.07422340282801); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:40:00', 110.51798004935188);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 14:50:00', 110.35858103803595); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:00:00', 110.9330824329225);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:10:00', 110.75721595824172); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:20:00', 110.16966145686376);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:30:00', 110.6200419734301); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:40:00', 110.02639206216864);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 15:50:00', 110.71466266056092); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:00:00', 110.42276135702156);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:10:00', 110.64703918748785); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:20:00', 110.00938878239657);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:30:00', 110.74005330603822); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:40:00', 110.35879570234194);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 16:50:00', 110.27423905054137); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:00:00', 110.57262870011722);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:10:00', 110.01985893324671); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:20:00', 110.01651001138652);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:30:00', 110.6343588062614); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:40:00', 110.7691057682859);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 17:50:00', 110.854592886564); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:00:00', 110.49398499569142);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:10:00', 110.22574642265914); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:20:00', 110.1989620219235);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:30:00', 110.11844858740491); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:40:00', 110.85888099958702);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 18:50:00', 110.72004125251655); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:00:00', 110.24675835750195);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:10:00', 110.29249904823641); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:20:00', 110.07829374409832);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:30:00', 110.47142972006851); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:40:00', 110.61157527376456);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 19:50:00', 110.57876731472328); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:00:00', 110.74785940159668);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:10:00', 110.10603151988887); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:20:00', 110.40784334254565);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:30:00', 110.73885667679576); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:40:00', 110.98592930132185);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 20:50:00', 110.03159173929218); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:00:00', 110.44003663363205);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:10:00', 110.23601033453232); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:20:00', 110.38588089796673);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:30:00', 110.70774915007118); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:40:00', 110.80238457151769);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 21:50:00', 110.2695060563233); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:00:00', 110.43972332144956);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:10:00', 110.30751590489595); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:20:00', 110.72538811804337);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:30:00', 110.29258672963152); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:40:00', 110.0362065937212);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 22:50:00', 110.67543648315056); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:00:00', 110.9021817678074);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:10:00', 110.31611102580412); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:20:00', 110.07205126887513);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:30:00', 110.30667208476088); +INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:40:00', 110.2096486408832);INSERT INTO ts_data (date, value) VALUES ('2016-12-14 23:50:00', 110.88528324901444); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:00:00', 110.51628474490116);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:10:00', 110.6566262439806); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:20:00', 110.6979105429185);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:30:00', 110.7173499344663); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:40:00', 110.13239608472007);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 00:50:00', 110.89353590114023); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:00:00', 110.09097519112751);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:10:00', 110.87044837143185); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:20:00', 110.65283407060114);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:30:00', 110.31014325206102); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:40:00', 110.02685067243759);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 01:50:00', 110.35220774702627); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:00:00', 110.00103050574343);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:10:00', 110.65596146610964); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:20:00', 110.08770330799145);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:30:00', 110.25361595070032); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:40:00', 110.42574750640628);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 02:50:00', 110.5237707383437); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:00:00', 110.56173231434866);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:10:00', 110.9703841087223); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:20:00', 110.27505286336691);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:30:00', 110.29340113028113); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:40:00', 110.32899197296658);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 03:50:00', 110.09626708999346); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:00:00', 115.92034509496644);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:10:00', 115.39922686791157); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:20:00', 115.05250578237221);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:30:00', 115.28072723909516); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:40:00', 115.50664108716694);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 04:50:00', 115.33329094145218); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:00:00', 115.33702847427412);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:10:00', 115.04981495129273); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:20:00', 115.05606091988008);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:30:00', 115.11740830894504); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:40:00', 115.4167621975373);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 05:50:00', 115.33128951639964); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:00:00', 115.62187401688368);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:10:00', 115.47647480832849); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:20:00', 115.4532001640554);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:30:00', 115.642196624224); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:40:00', 115.55298887565829);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 06:50:00', 115.49355051169212); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:00:00', 115.73059320995986);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:10:00', 115.76167513394451); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:20:00', 115.26306599022764);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:30:00', 115.51486681588571); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:40:00', 115.19562504740864);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 07:50:00', 115.17025430716672); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:00:00', 115.78013237676927);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:10:00', 115.03416311414273); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:20:00', 115.89882244794657);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:30:00', 115.81105607717902); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:40:00', 115.58443762971216);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 08:50:00', 115.26168713986871); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:00:00', 115.66526247866794);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:10:00', 115.82464119488087); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:20:00', 115.04056460532888);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:30:00', 115.98774330485566); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:40:00', 115.0104486595282);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 09:50:00', 115.96111537572357); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:00:00', 115.8245490214396);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:10:00', 115.38748286829578); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:20:00', 115.18520214522057);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:30:00', 115.3514485906698); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:40:00', 115.44665469221741);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 10:50:00', 115.55842156538937); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:00:00', 115.02560113526441);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:10:00', 115.87608037600435); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:20:00', 115.82420547970959);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:30:00', 115.01677905476782); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:40:00', 115.45409022327671);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 11:50:00', 115.12067421915043); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:00:00', 115.88727688865416);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:10:00', 115.52534722095196); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:20:00', 115.68163804179949);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:30:00', 115.539839218707); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:40:00', 115.22368930448073);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 12:50:00', 115.63567122140387); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:00:00', 115.60016350834304);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:10:00', 115.35732108719178); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:20:00', 115.74568200717638);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:30:00', 115.39704090173677); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:40:00', 115.61935157965172);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 13:50:00', 115.63610398605341); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:00:00', 115.89222340869463);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:10:00', 115.62902987838076); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:20:00', 115.52859321594143);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:30:00', 115.10513533273816); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:40:00', 115.88077983047351);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 14:50:00', 115.1130128018198); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:00:00', 115.98415202678764);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:10:00', 115.07473370436784); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:20:00', 115.3786367812187);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:30:00', 115.95605877330911); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:40:00', 115.89726054171273);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 15:50:00', 115.65894614508451); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:00:00', 115.24684375099228);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:10:00', 115.62974460698156); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:20:00', 115.69256150306074);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:30:00', 115.0166686892437); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:40:00', 115.3456336595683);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 16:50:00', 115.0234587345454); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:00:00', 115.97441992588934);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:10:00', 115.8448132541058); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:20:00', 115.65543028376051);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:30:00', 115.71208021448278); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:40:00', 115.4781607967377);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 17:50:00', 115.70510822555813); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:00:00', 115.52939212940808);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:10:00', 115.48561616524496); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:20:00', 115.52121044710016);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:30:00', 115.82674001086326); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:40:00', 115.20185675592418);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 18:50:00', 115.7595526052731); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:00:00', 115.40853540496208);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:10:00', 115.03500235867314); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:20:00', 115.13067419460987);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:30:00', 115.17327125512324); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:40:00', 115.64341194713968);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 19:50:00', 115.76641037060409); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:00:00', 115.83166136304564);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:10:00', 115.49761318146606); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:20:00', 115.45755696233412);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:30:00', 115.8410236957087); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:40:00', 120.82644489317774);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 20:50:00', 120.6549272826231); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:00:00', 120.08408232439287);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:10:00', 120.25385894689538); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:20:00', 120.831924185407);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:30:00', 120.80399119929008); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:40:00', 120.67437343753132);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 21:50:00', 120.56115061221644); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:00:00', 120.99023407744896);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:10:00', 120.65558185562443); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:20:00', 120.17956001975037);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:30:00', 120.72989876408664); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:40:00', 120.67663883423089);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 22:50:00', 120.98978608413451); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:00:00', 120.55715394264116);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:10:00', 120.3280233998929); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:20:00', 120.70094175757272);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:30:00', 120.98334063163055); +INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:40:00', 120.977634025921);INSERT INTO ts_data (date, value) VALUES ('2016-12-15 23:50:00', 120.19680092048293); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:00:00', 120.06476432921309);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:10:00', 120.31070585738662); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:20:00', 120.40696681089787);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:30:00', 120.60145851705968); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:40:00', 120.00420037302388);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 00:50:00', 120.03280696643525); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:00:00', 120.37692640276228);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:10:00', 120.36665173229142); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:20:00', 120.94387167130972);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:30:00', 120.19997124628559); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:40:00', 120.05592686663337);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 01:50:00', 120.8548233151384); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:00:00', 120.02699959879585);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:10:00', 120.8519105228424); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:20:00', 120.03283338082785);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:30:00', 120.32936547420351); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:40:00', 120.80212574372938);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 02:50:00', 120.23921574911108); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:00:00', 120.75504438897896);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:10:00', 120.86207940009143); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:20:00', 120.20937931299572);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:30:00', 120.30095044093979); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:40:00', 120.5030615421094);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 03:50:00', 120.28598019189259); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:00:00', 120.10827568073782);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:10:00', 120.98699785663374); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:20:00', 120.96268312288082);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:30:00', 120.82371301225443); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:40:00', 120.61830046723459);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 04:50:00', 120.79977611813331); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:00:00', 120.57291856612294);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:10:00', 120.98601099526851); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:20:00', 120.03862837254096);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:30:00', 120.17184224024064); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:40:00', 120.80091055324695);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 05:50:00', 120.34000838523704); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:00:00', 120.60809451938822);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:10:00', 120.29035379289171); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:20:00', 120.9040104301264);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:30:00', 120.31136665078724); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:40:00', 120.1812663625463);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 06:50:00', 120.93438651835392); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:00:00', 120.94577682649124);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:10:00', 120.55026693362673); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:20:00', 120.49869839960226);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:30:00', 120.76419846529501); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:40:00', 120.32891781041769);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 07:50:00', 120.89725128563713); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:00:00', 120.81384751975108);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:10:00', 120.42830276437893); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:20:00', 120.76397470598722);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:30:00', 120.81497089297474); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:40:00', 120.32003571451868);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 08:50:00', 120.15467173662675); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:00:00', 120.06726059846304);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:10:00', 120.82311834144716); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:20:00', 120.1820536253275);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:30:00', 120.21635293924656); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:40:00', 120.51710522608116);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 09:50:00', 120.82906762171484); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:00:00', 120.52113940697737);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:10:00', 120.84858280524145); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:20:00', 120.96207479146118);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:30:00', 120.44426662820351); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:40:00', 120.90808339360385);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 10:50:00', 120.74990586115463); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:00:00', 120.67549253070091);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:10:00', 120.3298160130022); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:20:00', 120.88633713175562);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:30:00', 120.96457496772064); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:40:00', 120.98077386733937);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 11:50:00', 120.576124391448); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:00:00', 120.34659036767653);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:10:00', 120.5856561536969); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:20:00', 120.52944343859464);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:30:00', 120.36055981032291); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:40:00', 120.95179568270576);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 12:50:00', 120.33330010157015); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:00:00', 120.85482930960468);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:10:00', 120.79264328111007); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:20:00', 125.65287280517842);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:30:00', 125.64396553535299); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:40:00', 125.71734437088135);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 13:50:00', 125.42390450174223); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:00:00', 125.47850363605784);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:10:00', 125.26349082218289); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:20:00', 125.23066891402209);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:30:00', 125.29121932583885); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:40:00', 125.40797519911968);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 14:50:00', 125.0081113243922); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:00:00', 125.73769673323004);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:10:00', 125.55333399980495); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:20:00', 125.9338480235524);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:30:00', 125.584996803726); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:40:00', 125.91354166413394);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 15:50:00', 125.0099316793555); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:00:00', 125.58488234330359);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:10:00', 125.86727983852556); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:20:00', 125.61582159763391);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:30:00', 125.50454381881113); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:40:00', 125.52156499547934);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 16:50:00', 125.77681738674585); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:00:00', 125.329974313302);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:10:00', 125.37982398964373); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:20:00', 125.2000700117828);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:30:00', 125.76333038634014); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:40:00', 125.1751994132285);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 17:50:00', 125.83473591020085); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:00:00', 125.90173521962282);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:10:00', 125.81451848262671); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:20:00', 125.25740759463517);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:30:00', 125.26163239499365); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:40:00', 125.71705306683364);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 18:50:00', 125.81613305294701); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:00:00', 125.10330966729616);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:10:00', 125.60604692287441); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:20:00', 125.45815200785763);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:30:00', 125.08557013990955); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:40:00', 125.97649026685646);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 19:50:00', 125.01208175825936); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:00:00', 125.73601458974932);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:10:00', 125.56786793153258); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:20:00', 125.43308043543266);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:30:00', 125.77946478349509); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:40:00', 125.28416932149958);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 20:50:00', 125.1822931183685); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:00:00', 125.41011613343115);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:10:00', 125.16691492182012); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:20:00', 125.59900937371857);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:30:00', 125.4239352559672); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:40:00', 125.36411003724287);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 21:50:00', 125.71835636953129); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:00:00', 125.14126768660087);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:10:00', 125.15172467772771); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:20:00', 125.86074627485101);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:30:00', 125.59779947517357); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:40:00', 125.82445111331042);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 22:50:00', 125.13686466249423); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:00:00', 125.53973678755672);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:10:00', 125.24667751804859); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:20:00', 125.33573809756126);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:30:00', 125.4419433566887); +INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:40:00', 125.83330875165787);INSERT INTO ts_data (date, value) VALUES ('2016-12-16 23:50:00', 125.22334573631028); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:00:00', 125.6414727679888);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:10:00', 125.55895516236252); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:20:00', 125.51127922981144);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:30:00', 125.24855436769563); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:40:00', 125.1715138490133);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 00:50:00', 125.90095334415271); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:00:00', 125.37706174808369);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:10:00', 125.72332978291209); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:20:00', 125.6247259341157);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:30:00', 125.77224995502243); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:40:00', 125.65948386671788);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 01:50:00', 125.94555285873936); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:00:00', 125.95422572214912);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:10:00', 125.8971838669447); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:20:00', 125.63645331661951);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:30:00', 125.92899146774526); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:40:00', 125.17840825816371);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 02:50:00', 125.31852650379227); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:00:00', 125.11065898281996);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:10:00', 125.61082397310557); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:20:00', 125.30540198466014);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:30:00', 125.84339428098431); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:40:00', 125.43905895299652);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 03:50:00', 125.23264337615092); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:00:00', 125.39144904762676);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:10:00', 125.47160455013405); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:20:00', 125.27470667067102);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:30:00', 125.1360922377444); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:40:00', 125.62329902316296);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 04:50:00', 125.66101845837649); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:00:00', 125.45546276762786);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:10:00', 125.43188406449724); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:20:00', 125.6870324820484);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:30:00', 125.6862262260098); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:40:00', 125.45986264549069);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 05:50:00', 125.87767184797652); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:00:00', 130.55752649889308);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:10:00', 130.47210457115082); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:20:00', 130.16083821591772);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:30:00', 130.0721132666919); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:40:00', 130.93182429388398);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 06:50:00', 130.88034006958418); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:00:00', 130.2960201404346);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:10:00', 130.81596206318963); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:20:00', 130.57712327914413);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:30:00', 130.9266257321872); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:40:00', 130.04945413971518);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 07:50:00', 130.2937922643273); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:00:00', 130.67992742849214);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:10:00', 130.52187784961671); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:20:00', 130.6815300385082);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:30:00', 130.1491898663459); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:40:00', 130.58692600595558);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 08:50:00', 130.59864771633175); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:00:00', 130.25205974444444);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:10:00', 130.51691670451189); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:20:00', 130.67374894414556);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:30:00', 130.8042564540647); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:40:00', 130.29362105587268);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 09:50:00', 130.77006018239035); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:00:00', 130.8588457916097);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:10:00', 130.31852233755862); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:20:00', 130.78765560716604);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:30:00', 130.54215704841198); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:40:00', 130.62851477591724);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 10:50:00', 130.72289371182964); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:00:00', 130.9918181823095);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:10:00', 130.42574240072523); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:20:00', 130.82852101016388);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:30:00', 130.4760645151284); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:40:00', 130.42018629271487);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 11:50:00', 130.1687714251814); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:00:00', 130.64233815481978);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:10:00', 130.49792908490352); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:20:00', 130.2525413864335);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:30:00', 130.77131974307093); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:40:00', 130.22991739697986);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 12:50:00', 130.91223647275896); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:00:00', 130.5269938909138);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:10:00', 130.22644406738803); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:20:00', 130.366184488456);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:30:00', 130.89817955425983); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:40:00', 130.0187264787575);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 13:50:00', 130.0602346252049); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:00:00', 130.08813722278666);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:10:00', 130.7376380926898); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:20:00', 130.11647754151107);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:30:00', 130.71971186904696); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:40:00', 130.26012816081476);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 14:50:00', 130.16294632396048); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:00:00', 130.89004024791146);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:10:00', 130.1610362610834); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:20:00', 130.76158826963186);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:30:00', 130.02155764039446); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:40:00', 130.06125995821742);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 15:50:00', 130.355675127596); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:00:00', 130.05219169492528);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:10:00', 130.92435792726826); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:20:00', 130.55939363652726);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:30:00', 130.94408625501342); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:40:00', 130.60133900877642);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 16:50:00', 130.5582953364659); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:00:00', 130.66243087111098);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:10:00', 130.1413215721195); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:20:00', 130.38149877843287);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:30:00', 130.3796352581514); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:40:00', 130.25363432472798);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 17:50:00', 130.43143986685902); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:00:00', 130.2389580537966);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:10:00', 130.66365461963042); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:20:00', 130.09582250337957);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:30:00', 130.8610290695083); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:40:00', 130.08579101152458);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 18:50:00', 130.9850413508669); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:00:00', 130.45040049483163);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:10:00', 130.96602251995662); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:20:00', 130.49534168094283);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:30:00', 130.69651652188273); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:40:00', 130.82244795778115);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 19:50:00', 130.23928203008177); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:00:00', 130.5342415952506);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:10:00', 130.86653257274222); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:20:00', 130.99315060435964);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:30:00', 130.70284782267896); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:40:00', 130.89953667800603);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 20:50:00', 130.59650905872653); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:00:00', 130.7439607481838);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:10:00', 130.73732494797144); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:20:00', 130.50496075815454);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:30:00', 130.86808847246445); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:40:00', 130.760322798989);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 21:50:00', 130.37022741605705); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:00:00', 130.8251499409934);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:10:00', 130.93879350865066); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:20:00', 130.42274793729106);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:30:00', 130.07145099626342); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:40:00', 135.5822275182252);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 22:50:00', 135.290875281548); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:00:00', 135.13915851315687);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:10:00', 135.01104907899665); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:20:00', 135.74063149714036);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:30:00', 135.9347280912333); +INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:40:00', 135.85710355684222);INSERT INTO ts_data (date, value) VALUES ('2016-12-17 23:50:00', 135.77097325941608); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:00:00', 135.8455559646272);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:10:00', 135.89470738995027); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:20:00', 135.2489775061593);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:30:00', 135.3793364271599); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:40:00', 135.38449742742876);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 00:50:00', 135.03700395403507); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:00:00', 135.7464313179241);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:10:00', 135.04886193497214); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:20:00', 135.82877631810058);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:30:00', 135.91725559542292); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:40:00', 135.16359170124807);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 01:50:00', 135.269706845287); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:00:00', 135.75926140801715);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:10:00', 135.273780989814); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:20:00', 135.11920099470748);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:30:00', 135.6458249517251); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:40:00', 135.83506575523225);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 02:50:00', 135.1842346591327); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:00:00', 135.9559008321893);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:10:00', 135.71165753175615); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:20:00', 135.11854473865588);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:30:00', 135.56454284079038); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:40:00', 135.04791114699213);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 03:50:00', 135.66121579862698); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:00:00', 135.97693668813005);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:10:00', 135.01957571131328); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:20:00', 135.1784325684813);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:30:00', 135.9271176272339); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:40:00', 135.50456868397166);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 04:50:00', 135.18801922909378); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:00:00', 135.11824025540568);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:10:00', 135.12655456301354); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:20:00', 135.242658101112);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:30:00', 135.33625561865995); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:40:00', 135.5751045589858);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 05:50:00', 135.10703908836848); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:00:00', 135.49237126922029);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:10:00', 135.12773622383622); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:20:00', 135.48231168086377);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:30:00', 135.45502900632232); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:40:00', 135.31825332626258);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 06:50:00', 135.73105659175795); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:00:00', 135.25554367703347);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:10:00', 135.30102734148892); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:20:00', 135.1051556273559);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:30:00', 135.04007777331657); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:40:00', 135.77849168864321);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 07:50:00', 135.63694303307958); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:00:00', 135.74603303295677);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:10:00', 135.967451799846); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:20:00', 135.99283924105453);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:30:00', 135.38215001155572); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:40:00', 135.7203834940102);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 08:50:00', 135.72776660324354); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:00:00', 135.40006510083572);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:10:00', 135.18858626175364); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:20:00', 135.95836122705143);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:30:00', 135.25379951813704); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:40:00', 135.7276495111565);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 09:50:00', 135.49940105799575); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:00:00', 135.04478992920576);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:10:00', 135.16158472057415); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:20:00', 135.19104124900028);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:30:00', 135.1978962720263); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:40:00', 135.8797741927591);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 10:50:00', 135.42893695793265); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:00:00', 135.75721654223466);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:10:00', 135.1147303536879); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:20:00', 135.46564010969075);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:30:00', 135.28025985628759); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:40:00', 135.50777684605671);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 11:50:00', 135.85619380336865); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:00:00', 135.3321464077477);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:10:00', 135.76739484688176); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:20:00', 135.97454562186522);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:30:00', 135.23968953026392); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:40:00', 135.0013359421395);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 12:50:00', 135.14294928179038); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:00:00', 135.36118593983392);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:10:00', 135.01275592755974); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:20:00', 135.53935038333822);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:30:00', 135.06951475453644); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:40:00', 135.47701969383752);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 13:50:00', 135.49299753337226); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:00:00', 135.71005467665705);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:10:00', 135.6890138348129); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:20:00', 135.8286072166702);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:30:00', 135.7940921069671); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:40:00', 135.09662324353906);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 14:50:00', 135.01061425064736); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:00:00', 135.76969936568676);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:10:00', 135.02477095323124); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:20:00', 140.5524095544186);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:30:00', 140.38860659330004); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:40:00', 140.01734910815307);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 15:50:00', 140.1597221181258); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:00:00', 140.0314899124773);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:10:00', 140.1008617715402); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:20:00', 140.84218644510938);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:30:00', 140.95380320925605); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:40:00', 140.02786801928954);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 16:50:00', 140.68615634959215); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:00:00', 140.1455071617251);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:10:00', 140.66160049491523); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:20:00', 140.45608688522498);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:30:00', 140.45648268027682); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:40:00', 140.97569589085234);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 17:50:00', 140.67644041617714); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:00:00', 140.65864095209272);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:10:00', 140.52405153234466); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:20:00', 140.86515045160368);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:30:00', 140.66659420562374); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:40:00', 140.57133784911477);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 18:50:00', 140.16110513816938); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:00:00', 140.8304104307717);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:10:00', 140.78310964285572); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:20:00', 140.05597285732236);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:30:00', 140.70485364399892); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:40:00', 140.93107006701882);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 19:50:00', 140.57346515491787); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:00:00', 140.98225769949184);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:10:00', 140.32615795074054); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:20:00', 140.12590509899022);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:30:00', 140.70668039065674); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:40:00', 140.4140851064422);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 20:50:00', 140.03313073052286); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:00:00', 140.63351926697442);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:10:00', 140.1829796940263); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:20:00', 140.47967597934021);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:30:00', 140.34413142548922); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:40:00', 140.0426108415719);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 21:50:00', 140.08335049468613); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:00:00', 140.5809851312373);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:10:00', 140.30746319215652); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:20:00', 140.4437908140055);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:30:00', 140.02014453174584); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:40:00', 140.2128779109799);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 22:50:00', 140.5180134001954); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:00:00', 140.8832518623758);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:10:00', 140.72660708365876); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:20:00', 140.13260916673494);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:30:00', 140.6685779559208); +INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:40:00', 140.58626918907706);INSERT INTO ts_data (date, value) VALUES ('2016-12-18 23:50:00', 140.76837168351068); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:00:00', 140.88943196987566);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:10:00', 140.3192286564333); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:20:00', 140.76723922879268);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:30:00', 140.5477150242652); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:40:00', 140.50047597246095);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 00:50:00', 140.59671771658958); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:00:00', 140.760457655149);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:10:00', 140.75917327120624); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:20:00', 140.01496604863922);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:30:00', 140.29081096369455); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:40:00', 140.67957024313557);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 01:50:00', 140.43814891956498); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:00:00', 140.47525383039314);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:10:00', 140.9418632322256); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:20:00', 140.76196779867192);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:30:00', 140.56609907176457); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:40:00', 140.85769612649491);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 02:50:00', 140.75518382700537); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:00:00', 140.01234263559613);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:10:00', 140.40596437354662); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:20:00', 140.48195068926188);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:30:00', 140.50113518688178); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:40:00', 140.2537437243218);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 03:50:00', 140.57588910299367); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:00:00', 140.5011731953);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:10:00', 140.32275275368934); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:20:00', 140.33672467424947);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:30:00', 140.6486065897093); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:40:00', 140.2221530753053);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 04:50:00', 140.28903592783888); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:00:00', 140.37179537480162);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:10:00', 140.50654885088002); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:20:00', 140.23551619652952);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:30:00', 140.5445764128223); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:40:00', 140.8115348719777);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 05:50:00', 140.13747874955092); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:00:00', 140.11103895370636);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:10:00', 140.77433273235442); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:20:00', 140.29246854553352);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:30:00', 140.25120631632427); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:40:00', 140.4406017165515);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 06:50:00', 140.58915042935652); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:00:00', 140.3146642494925);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:10:00', 140.28985323828846); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:20:00', 140.65486025792748);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:30:00', 140.05970824708018); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:40:00', 140.77085915678708);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 07:50:00', 140.22804689656098); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:00:00', 145.66987523875392);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:10:00', 145.1707350865351); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:20:00', 145.16940803645684);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:30:00', 145.28901088567568); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:40:00', 145.4980042071863);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 08:50:00', 145.2205579877147); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:00:00', 145.69597916914574);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:10:00', 145.58124546129085); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:20:00', 145.8574157476726);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:30:00', 145.6369219223971); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:40:00', 145.78766875330268);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 09:50:00', 145.35171586860537); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:00:00', 145.7281389158997);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:10:00', 145.30110340970037); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:20:00', 145.55264598747647);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:30:00', 145.60658471565296); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:40:00', 145.52161260399842);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 10:50:00', 145.96643519774298); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:00:00', 145.82570031315652);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:10:00', 145.25411620623473); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:20:00', 145.01084066406216);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:30:00', 145.765867810619); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:40:00', 145.89444272758473);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 11:50:00', 145.77793331580872); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:00:00', 145.44588175007783);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:10:00', 145.49194633436278); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:20:00', 145.30017295055927);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:30:00', 145.41543052738336); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:40:00', 145.8074991727321);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 12:50:00', 145.12258075325624); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:00:00', 145.72481690483806);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:10:00', 145.7129788820173); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:20:00', 145.2621944801272);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:30:00', 145.8563327369375); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:40:00', 145.11773145469112);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 13:50:00', 145.94891778385843); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:00:00', 145.34019456698357);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:10:00', 145.65932709875548); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:20:00', 145.43751530410034);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:30:00', 145.0593460373174); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:40:00', 145.88811740693234);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 14:50:00', 145.90368066651718); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:00:00', 145.95884728201386);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:10:00', 145.06150896281636); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:20:00', 145.8622261033162);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:30:00', 145.83755664275444); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:40:00', 145.43972995819828);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 15:50:00', 145.73981228170072); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:00:00', 145.12681461732356);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:10:00', 145.012754868371); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:20:00', 145.0505322143747);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:30:00', 145.88282679938504); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:40:00', 145.00081151155172);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 16:50:00', 145.43198725370831); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:00:00', 145.82429510444118);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:10:00', 145.9498559447054); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:20:00', 145.97480554095102);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:30:00', 145.4446798624002); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:40:00', 145.2216477801595);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 17:50:00', 145.37234684968098); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:00:00', 145.0455193928619);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:10:00', 145.57137581029124); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:20:00', 145.97361696560858);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:30:00', 145.415751650966); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:40:00', 145.08356782962744);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 18:50:00', 145.88023605798838); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:00:00', 145.40523929500432);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:10:00', 145.3418164246789); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:20:00', 145.59691692201514);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:30:00', 145.4793346319479); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:40:00', 145.64966501406303);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 19:50:00', 145.50216746294907); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:00:00', 145.91907700356387);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:10:00', 145.87403548263796); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:20:00', 145.46946986838512);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:30:00', 145.4998155317617); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:40:00', 145.27447045101775);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 20:50:00', 145.52002992975224); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:00:00', 145.91042830526916);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:10:00', 145.5409829527971); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:20:00', 145.63202944838724);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:30:00', 145.5344594614299); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:40:00', 145.41555312632602);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 21:50:00', 145.13658569771513); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:00:00', 145.8566582624446);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:10:00', 145.81325233346422); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:20:00', 145.78274734773234);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:30:00', 145.0640730008673); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:40:00', 145.80812656708284);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 22:50:00', 145.87468388022162); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:00:00', 145.67072629238953);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:10:00', 145.88096610600363); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:20:00', 145.09472335198168);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:30:00', 145.6638566197089); +INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:40:00', 145.08440919685958);INSERT INTO ts_data (date, value) VALUES ('2016-12-19 23:50:00', 145.44861743475232); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:00:00', 145.89047384558398);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:10:00', 145.27914766231697); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:20:00', 145.69446146340147);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:30:00', 145.93709895795772); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:40:00', 150.95389038965732);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 00:50:00', 150.4695271916118); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:00:00', 150.71143300502908);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:10:00', 150.06902295783496); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:20:00', 150.94789146885924);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:30:00', 150.02560775759667); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:40:00', 150.46498881080538);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 01:50:00', 150.0295339298049); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:00:00', 150.9363084671751);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:10:00', 150.49309095688454); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:20:00', 150.42234840916416);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:30:00', 150.60626264187394); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:40:00', 150.29938954041432);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 02:50:00', 150.40643772440583); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:00:00', 150.29224512392312);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:10:00', 150.2138297054897); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:20:00', 150.74630354105142);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:30:00', 150.6907079934565); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:40:00', 150.03038540811488);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 03:50:00', 150.21958513347758); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:00:00', 150.17823386638824);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:10:00', 150.50507478083375); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:20:00', 150.79508618235164);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:30:00', 150.13488288529962); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:40:00', 150.1130873537007);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 04:50:00', 150.43758652726063); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:00:00', 150.5954318958787);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:10:00', 150.63272477805202); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:20:00', 150.85457355613744);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:30:00', 150.21404408605937); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:40:00', 150.34826903440288);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 05:50:00', 150.1586167071405); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:00:00', 150.21425811542989);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:10:00', 150.0486196281555); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:20:00', 150.33931288773363);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:30:00', 150.1331451778826); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:40:00', 150.3179776855876);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 06:50:00', 150.6026645961371); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:00:00', 150.63000440845244);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:10:00', 150.26746307195071); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:20:00', 150.30597385822765);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:30:00', 150.71097930825889); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:40:00', 150.2119519323343);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 07:50:00', 150.95334503876776); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:00:00', 150.21115641274213);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:10:00', 150.4672349421686); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:20:00', 150.35080350412545);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:30:00', 150.4910943472531); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:40:00', 150.6225395747);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 08:50:00', 150.89177789608738); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:00:00', 150.5461716161229);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:10:00', 150.63226578869234); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:20:00', 150.85689962949857);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:30:00', 150.9448742016535); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:40:00', 150.92408268138675);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 09:50:00', 150.73421350377888); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:00:00', 150.3034589100252);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:10:00', 150.7147463476092); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:20:00', 150.4921060710854);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:30:00', 150.0829502967536); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:40:00', 150.81063367796378);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 10:50:00', 150.36401854193718); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:00:00', 150.35446335237458);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:10:00', 150.23659343990658); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:20:00', 150.9489373250949);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:30:00', 150.72168461362784); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:40:00', 150.98344616973552);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 11:50:00', 150.90325970412428); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:00:00', 150.6322081841063);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:10:00', 150.07288034629926); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:20:00', 150.2769904157787);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:30:00', 150.45689512291858); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:40:00', 150.8623556168554);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 12:50:00', 150.65527174054034); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:00:00', 150.4105151231106);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:10:00', 150.3992746961309); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:20:00', 150.2225099991635);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:30:00', 150.6253682325025); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:40:00', 150.15362327609586);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 13:50:00', 150.45290266139548); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:00:00', 150.80716539465936);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:10:00', 150.53582530927787); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:20:00', 150.03001506957418);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:30:00', 150.29875278657747); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:40:00', 150.0375390473902);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 14:50:00', 150.30909323458138); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:00:00', 150.9957959535933);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:10:00', 150.5915194444149); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:20:00', 150.37621528200074);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:30:00', 150.93192625156226); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:40:00', 150.40413833750725);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 15:50:00', 150.49369775455642); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:00:00', 150.17480087886327);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:10:00', 150.30585649477317); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:20:00', 150.44827829973516);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:30:00', 150.1010045943311); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:40:00', 150.778458080293);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 16:50:00', 150.91295092252244); +INSERT INTO ts_data (date, value) VALUES ('2016-12-20 17:00:00', 150.09800267705262);INSERT INTO ts_data (date, value) VALUES ('2016-12-20 17:10:00', 150.28707660560792); + + +DROP TABLE IF EXISTS prophet_model; +SELECT MADLIB_SCHEMA.prophet_fit('ts_data', 'prophet_model', $${'ds':'date', 'y': 'value'}$$, $${'growth': 'linear' , 'changepoints': ['2016-11-30 00:20:00', '2016-11-30 05:40:00']}$$, True); +SELECT ASSERT(count(*) = 1, 'prophet: prophet parameter counts error') FROM prophet_model; + +DROP TABLE IF EXISTS prophet_output; +SELECT MADLIB_SCHEMA.prophet_predict('prophet_model', 'prophet_output', 1008, '10T', NULL, NULL, False); + diff --git a/src/ports/postgres/modules/facebook_prophet/test/prophet_test2.sql_in b/src/ports/postgres/modules/facebook_prophet/test/prophet_test2.sql_in new file mode 100644 index 0000000..3d02aa2 --- /dev/null +++ b/src/ports/postgres/modules/facebook_prophet/test/prophet_test2.sql_in @@ -0,0 +1,99 @@ +/* +* Test case of facebook-prophet +*/ + +m4_include(`SQLCommon.m4') + +select MADLIB_SCHEMA.prophet_fit(); +select MADLIB_SCHEMA.prophet_fit('usage'); + +DROP TABLE IF EXISTS ts_data2; +CREATE TABLE ts_data2 (date date, value float); + + +INSERT INTO ts_data2 (date, value) VALUES ('1949-01-01', 112);INSERT INTO ts_data2 (date, value) VALUES ('1949-02-01', 118); +INSERT INTO ts_data2 (date, value) VALUES ('1949-03-01', 132);INSERT INTO ts_data2 (date, value) VALUES ('1949-04-01', 129); +INSERT INTO ts_data2 (date, value) VALUES ('1949-05-01', 121);INSERT INTO ts_data2 (date, value) VALUES ('1949-06-01', 135); +INSERT INTO ts_data2 (date, value) VALUES ('1949-07-01', 148);INSERT INTO ts_data2 (date, value) VALUES ('1949-08-01', 148); +INSERT INTO ts_data2 (date, value) VALUES ('1949-09-01', 136);INSERT INTO ts_data2 (date, value) VALUES ('1949-10-01', 119); +INSERT INTO ts_data2 (date, value) VALUES ('1949-11-01', 104);INSERT INTO ts_data2 (date, value) VALUES ('1949-12-01', 118); +INSERT INTO ts_data2 (date, value) VALUES ('1950-01-01', 115);INSERT INTO ts_data2 (date, value) VALUES ('1950-02-01', 126); +INSERT INTO ts_data2 (date, value) VALUES ('1950-03-01', 141);INSERT INTO ts_data2 (date, value) VALUES ('1950-04-01', 135); +INSERT INTO ts_data2 (date, value) VALUES ('1950-05-01', 125);INSERT INTO ts_data2 (date, value) VALUES ('1950-06-01', 149); +INSERT INTO ts_data2 (date, value) VALUES ('1950-07-01', 170);INSERT INTO ts_data2 (date, value) VALUES ('1950-08-01', 170); +INSERT INTO ts_data2 (date, value) VALUES ('1950-09-01', 158);INSERT INTO ts_data2 (date, value) VALUES ('1950-10-01', 133); +INSERT INTO ts_data2 (date, value) VALUES ('1950-11-01', 114);INSERT INTO ts_data2 (date, value) VALUES ('1950-12-01', 140); +INSERT INTO ts_data2 (date, value) VALUES ('1951-01-01', 145);INSERT INTO ts_data2 (date, value) VALUES ('1951-02-01', 150); +INSERT INTO ts_data2 (date, value) VALUES ('1951-03-01', 178);INSERT INTO ts_data2 (date, value) VALUES ('1951-04-01', 163); +INSERT INTO ts_data2 (date, value) VALUES ('1951-05-01', 172);INSERT INTO ts_data2 (date, value) VALUES ('1951-06-01', 178); +INSERT INTO ts_data2 (date, value) VALUES ('1951-07-01', 199);INSERT INTO ts_data2 (date, value) VALUES ('1951-08-01', 199); +INSERT INTO ts_data2 (date, value) VALUES ('1951-09-01', 184);INSERT INTO ts_data2 (date, value) VALUES ('1951-10-01', 162); +INSERT INTO ts_data2 (date, value) VALUES ('1951-11-01', 146);INSERT INTO ts_data2 (date, value) VALUES ('1951-12-01', 166); +INSERT INTO ts_data2 (date, value) VALUES ('1952-01-01', 171);INSERT INTO ts_data2 (date, value) VALUES ('1952-02-01', 180); +INSERT INTO ts_data2 (date, value) VALUES ('1952-03-01', 193);INSERT INTO ts_data2 (date, value) VALUES ('1952-04-01', 181); +INSERT INTO ts_data2 (date, value) VALUES ('1952-05-01', 183);INSERT INTO ts_data2 (date, value) VALUES ('1952-06-01', 218); +INSERT INTO ts_data2 (date, value) VALUES ('1952-07-01', 230);INSERT INTO ts_data2 (date, value) VALUES ('1952-08-01', 242); +INSERT INTO ts_data2 (date, value) VALUES ('1952-09-01', 209);INSERT INTO ts_data2 (date, value) VALUES ('1952-10-01', 191); +INSERT INTO ts_data2 (date, value) VALUES ('1952-11-01', 172);INSERT INTO ts_data2 (date, value) VALUES ('1952-12-01', 194); +INSERT INTO ts_data2 (date, value) VALUES ('1953-01-01', 196);INSERT INTO ts_data2 (date, value) VALUES ('1953-02-01', 196); +INSERT INTO ts_data2 (date, value) VALUES ('1953-03-01', 236);INSERT INTO ts_data2 (date, value) VALUES ('1953-04-01', 235); +INSERT INTO ts_data2 (date, value) VALUES ('1953-05-01', 229);INSERT INTO ts_data2 (date, value) VALUES ('1953-06-01', 243); +INSERT INTO ts_data2 (date, value) VALUES ('1953-07-01', 264);INSERT INTO ts_data2 (date, value) VALUES ('1953-08-01', 272); +INSERT INTO ts_data2 (date, value) VALUES ('1953-09-01', 237);INSERT INTO ts_data2 (date, value) VALUES ('1953-10-01', 211); +INSERT INTO ts_data2 (date, value) VALUES ('1953-11-01', 180);INSERT INTO ts_data2 (date, value) VALUES ('1953-12-01', 201); +INSERT INTO ts_data2 (date, value) VALUES ('1954-01-01', 204);INSERT INTO ts_data2 (date, value) VALUES ('1954-02-01', 188); +INSERT INTO ts_data2 (date, value) VALUES ('1954-03-01', 235);INSERT INTO ts_data2 (date, value) VALUES ('1954-04-01', 227); +INSERT INTO ts_data2 (date, value) VALUES ('1954-05-01', 234);INSERT INTO ts_data2 (date, value) VALUES ('1954-06-01', 264); +INSERT INTO ts_data2 (date, value) VALUES ('1954-07-01', 302);INSERT INTO ts_data2 (date, value) VALUES ('1954-08-01', 293); +INSERT INTO ts_data2 (date, value) VALUES ('1954-09-01', 259);INSERT INTO ts_data2 (date, value) VALUES ('1954-10-01', 229); +INSERT INTO ts_data2 (date, value) VALUES ('1954-11-01', 203);INSERT INTO ts_data2 (date, value) VALUES ('1954-12-01', 229); +INSERT INTO ts_data2 (date, value) VALUES ('1955-01-01', 242);INSERT INTO ts_data2 (date, value) VALUES ('1955-02-01', 233); +INSERT INTO ts_data2 (date, value) VALUES ('1955-03-01', 267);INSERT INTO ts_data2 (date, value) VALUES ('1955-04-01', 269); +INSERT INTO ts_data2 (date, value) VALUES ('1955-05-01', 270);INSERT INTO ts_data2 (date, value) VALUES ('1955-06-01', 315); +INSERT INTO ts_data2 (date, value) VALUES ('1955-07-01', 364);INSERT INTO ts_data2 (date, value) VALUES ('1955-08-01', 347); +INSERT INTO ts_data2 (date, value) VALUES ('1955-09-01', 312);INSERT INTO ts_data2 (date, value) VALUES ('1955-10-01', 274); +INSERT INTO ts_data2 (date, value) VALUES ('1955-11-01', 237);INSERT INTO ts_data2 (date, value) VALUES ('1955-12-01', 278); +INSERT INTO ts_data2 (date, value) VALUES ('1956-01-01', 284);INSERT INTO ts_data2 (date, value) VALUES ('1956-02-01', 277); +INSERT INTO ts_data2 (date, value) VALUES ('1956-03-01', 317);INSERT INTO ts_data2 (date, value) VALUES ('1956-04-01', 313); +INSERT INTO ts_data2 (date, value) VALUES ('1956-05-01', 318);INSERT INTO ts_data2 (date, value) VALUES ('1956-06-01', 374); +INSERT INTO ts_data2 (date, value) VALUES ('1956-07-01', 413);INSERT INTO ts_data2 (date, value) VALUES ('1956-08-01', 405); +INSERT INTO ts_data2 (date, value) VALUES ('1956-09-01', 355);INSERT INTO ts_data2 (date, value) VALUES ('1956-10-01', 306); +INSERT INTO ts_data2 (date, value) VALUES ('1956-11-01', 271);INSERT INTO ts_data2 (date, value) VALUES ('1956-12-01', 306); +INSERT INTO ts_data2 (date, value) VALUES ('1957-01-01', 315);INSERT INTO ts_data2 (date, value) VALUES ('1957-02-01', 301); +INSERT INTO ts_data2 (date, value) VALUES ('1957-03-01', 356);INSERT INTO ts_data2 (date, value) VALUES ('1957-04-01', 348); +INSERT INTO ts_data2 (date, value) VALUES ('1957-05-01', 355);INSERT INTO ts_data2 (date, value) VALUES ('1957-06-01', 422); +INSERT INTO ts_data2 (date, value) VALUES ('1957-07-01', 465);INSERT INTO ts_data2 (date, value) VALUES ('1957-08-01', 467); +INSERT INTO ts_data2 (date, value) VALUES ('1957-09-01', 404);INSERT INTO ts_data2 (date, value) VALUES ('1957-10-01', 347); +INSERT INTO ts_data2 (date, value) VALUES ('1957-11-01', 305);INSERT INTO ts_data2 (date, value) VALUES ('1957-12-01', 336); +INSERT INTO ts_data2 (date, value) VALUES ('1958-01-01', 340);INSERT INTO ts_data2 (date, value) VALUES ('1958-02-01', 318); +INSERT INTO ts_data2 (date, value) VALUES ('1958-03-01', 362);INSERT INTO ts_data2 (date, value) VALUES ('1958-04-01', 348); +INSERT INTO ts_data2 (date, value) VALUES ('1958-05-01', 363);INSERT INTO ts_data2 (date, value) VALUES ('1958-06-01', 435); +INSERT INTO ts_data2 (date, value) VALUES ('1958-07-01', 491);INSERT INTO ts_data2 (date, value) VALUES ('1958-08-01', 505); +INSERT INTO ts_data2 (date, value) VALUES ('1958-09-01', 404);INSERT INTO ts_data2 (date, value) VALUES ('1958-10-01', 359); +INSERT INTO ts_data2 (date, value) VALUES ('1958-11-01', 310);INSERT INTO ts_data2 (date, value) VALUES ('1958-12-01', 337); +INSERT INTO ts_data2 (date, value) VALUES ('1959-01-01', 360);INSERT INTO ts_data2 (date, value) VALUES ('1959-02-01', 342); +INSERT INTO ts_data2 (date, value) VALUES ('1959-03-01', 406);INSERT INTO ts_data2 (date, value) VALUES ('1959-04-01', 396); +INSERT INTO ts_data2 (date, value) VALUES ('1959-05-01', 420);INSERT INTO ts_data2 (date, value) VALUES ('1959-06-01', 472); +INSERT INTO ts_data2 (date, value) VALUES ('1959-07-01', 548);INSERT INTO ts_data2 (date, value) VALUES ('1959-08-01', 559); +INSERT INTO ts_data2 (date, value) VALUES ('1959-09-01', 463);INSERT INTO ts_data2 (date, value) VALUES ('1959-10-01', 407); +INSERT INTO ts_data2 (date, value) VALUES ('1959-11-01', 362);INSERT INTO ts_data2 (date, value) VALUES ('1959-12-01', 405); +INSERT INTO ts_data2 (date, value) VALUES ('1960-01-01', 417);INSERT INTO ts_data2 (date, value) VALUES ('1960-02-01', 391); +INSERT INTO ts_data2 (date, value) VALUES ('1960-03-01', 419);INSERT INTO ts_data2 (date, value) VALUES ('1960-04-01', 461); +INSERT INTO ts_data2 (date, value) VALUES ('1960-05-01', 472);INSERT INTO ts_data2 (date, value) VALUES ('1960-06-01', 535); +INSERT INTO ts_data2 (date, value) VALUES ('1960-07-01', 622);INSERT INTO ts_data2 (date, value) VALUES ('1960-08-01', 606); +INSERT INTO ts_data2 (date, value) VALUES ('1960-09-01', 508);INSERT INTO ts_data2 (date, value) VALUES ('1960-10-01', 461); +INSERT INTO ts_data2 (date, value) VALUES ('1960-11-01', 390);INSERT INTO ts_data2 (date, value) VALUES ('1960-12-01', 432); + + +ALTER TABLE ts_data2 ADD COLUMN cap FLOAT; +ALTER TABLE ts_data2 ADD COLUMN floor FLOAT; +UPDATE ts_data2 SET cap=600; +UPDATE ts_data2 SET floor=0; + +DROP TABLE IF EXISTS prophet_model2; +SELECT MADLIB_SCHEMA.prophet_fit('ts_data2', 'prophet_model2', $${'ds':'date', 'y': 'value', 'cap': 'cap', 'floor': 'floor'}$$, $${'growth': 'logistic'}$$, True); +SELECT ASSERT(count(*) = 1, 'prophet: prophet parameter counts error') FROM prophet_model2; + +DROP TABLE IF EXISTS prophet_output; +SELECT MADLIB_SCHEMA.prophet_predict('prophet_model2', 'prophet_output2', 10, '1D', 600, 0, False); + diff --git a/src/ports/postgres/modules/gbdt/__init__.py_in b/src/ports/postgres/modules/gbdt/__init__.py_in new file mode 100644 index 0000000..e69de29 diff --git a/src/ports/postgres/modules/gbdt/gbdt.py_in b/src/ports/postgres/modules/gbdt/gbdt.py_in new file mode 100644 index 0000000..4004c65 --- /dev/null +++ b/src/ports/postgres/modules/gbdt/gbdt.py_in @@ -0,0 +1,306 @@ +# -*- coding:utf-8 -*- +# Copyright (c) 2020 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- + +import numpy as np +import plpy +from math import sqrt, ceil + +from utilities.control import MinWarning +from utilities.utilities import _assert +from utilities.utilities import extract_keyvalue_params +from utilities.validate_args import get_expr_type + +from recursive_partitioning.decision_tree import _tree_train_using_bins +from recursive_partitioning.decision_tree import _get_features_to_use +from recursive_partitioning.decision_tree import _classify_features +from recursive_partitioning.decision_tree import _get_filter_str +from recursive_partitioning.decision_tree import _get_bins +from recursive_partitioning.decision_tree import get_feature_str + + + +def gdbt_mean_square_error(y_pred, y): + y_pred = np.reshape(y_pred,[-1]) + y = np.reshape(y,[-1]) + return 0.5 * np.square(y_pred - y) / y.shape[0] + +def create_model_table(schema_madlib,output_table_name,tree_state, bins, tree_iter): + head = '' + if tree_iter == 0: + head = """CREATE TABLE {output_table_name} AS""".format(output_table_name = output_table_name) + else: + head = "INSERT INTO " + output_table_name + + depth = (tree_state['pruned_depth'] if 'pruned_depth' in tree_state + else tree_state['tree_depth']) + + sql = head + """ SELECT {tree_iter} as iteration, + $1 AS tree, + $2 as cat_levels_in_text, + $3 as cat_n_levels, + {depth} as tree_depth + """.format(tree_iter = tree_iter, depth=depth) + + sql_plan = plpy.prepare(sql, ['{0}.bytea8'.format(schema_madlib), 'text[]','integer[]']) + plpy.execute(sql_plan,[tree_state['tree_state'],bins['cat_origin'],bins['cat_n']]) + return None + +def create_summary_table(output_table, null_proxy, cat_f, con_f, learning_rate, is_classification, predict_dt_prob, num_trees, source_table): + summary_table_name = output_table + '_summary' + null_proxy_str="NULL" if null_proxy is None else "'{0}'".format(null_proxy) + cat_f = ','.join(cat_f) + con_f = ','.join(con_f) + create_sql = """CREATE TABLE {summary_table_name} AS SELECT + 'GBDT'::TEXT as method, + '{cat_f}'::text AS cat_features, + '{con_f}'::text AS con_features, + '{source_table}'::text AS source_table, + '{output_table}'::text AS model_table, + {null_proxy_str}::text AS null_proxy, + {learning_rate}::double precision AS learning_rate, + {is_classification}::BOOLEAN AS is_classification, + '{predict_dt_prob}'::text AS predict_dt_prob, + {num_trees}::INTEGER AS num_trees + """.format(**locals()) + plpy.execute(create_sql) + +def create_test_table(test_result_table, id, y_pred): + plpy.execute( """CREATE TABLE {test_result_table} AS SELECT + unnest(ARRAY{id}) as id, + unnest(ARRAY{y_pred}) as test_prediction + """.format(**locals())) + +def tree_fit(schema_madlib, bins, training_table_name, cat_features, con_features, + boolean_cats, num_bins,weights, + dep, min_split, min_bucket, + max_tree_depth, filter_null, dep_n_levels, + split_criterion, + num_random_features, + max_n_surr,null_proxy,verbose_mode=False,**kwargs): + + msg_level = "notice" if verbose_mode else "warning" + + with MinWarning(msg_level): + tree = _tree_train_using_bins( + schema_madlib, bins, training_table_name, cat_features, con_features, + boolean_cats, num_bins, weights, dep, min_split, + min_bucket, max_tree_depth, filter_null, dep_n_levels, + False, split_criterion, True, + num_random_features, max_n_surr, null_proxy) + + tree['grp_key'] = ' ' + return tree + +def tree_gradient_predict(schema_madlib, model_table,iteration, id_col_name, cat_features_str, + con_features_str, is_classification, predict_dt_prob, source_table): + id = [] + res = [] + + if not is_classification or (predict_dt_prob == "response"): + sql = """ SELECT + {id_col_name} as id, + {schema_madlib}._predict_dt_response( + tree, + {cat_features_str}::INTEGER[], + {con_features_str}::DOUBLE PRECISION[]) as gradient_prediction + FROM ( + SELECT * FROM {model_table} + WHERE iteration = {iteration}) as m, + {source_table}""".format(**locals()) + else : + sql = """ SELECT + {id_col_name} as id, + {schema_madlib}._predict_dt_prob( + tree, + {cat_features_str}::INTEGER[], + {con_features_str}::DOUBLE PRECISION[]) as gradient_prediction + FROM ( + SELECT * FROM {model_table} + WHERE iteration = {iteration}) as m, + {source_table}""".format(**locals()) + rows = plpy.execute(sql) + for r in rows: + id.append(r['id']) + res.append(r['gradient_prediction']) + return id , res + +def _get_gradient(y, y_pred, id, iteration, y_table_name): + # 1.calculate the loss + compute_loss = gdbt_mean_square_error(y_pred, y) + + # 2.calculate the gradient + cal_gradient = (np.gradient(compute_loss)).tolist() + + # 3.add column called gradient to source_table + if iteration == 1: + addCol_sql = """alter table {y_table_name} add gradient double precision""".format(**locals()) + plpy.execute(addCol_sql) + + update_sql = """UPDATE {y_table_name} SET gradient = temp_gradient FROM (SELECT unnest(ARRAY{cal_gradient}) as temp_gradient, + unnest(ARRAY{id}) as temp_id)q WHERE id = temp_id""".format(**locals()) + plpy.execute(update_sql) + return cal_gradient + +def gbdt_fit(schema_madlib,training_table_name, output_table_name, + id_col_name, dependent_variable, list_of_features, + list_of_features_to_exclude, weights, + num_trees, num_random_features, + max_tree_depth, min_split, min_bucket, num_bins, + null_handling_params, is_classification, + predict_dt_prob = None, learning_rate = None, + verbose=False, **kwargs): + + # 1. default paremeters + num_trees = 100 if num_trees is None else num_trees + weights = '1' if not weights or not weights.strip() else weights.strip() + max_tree_depth = 10 if max_tree_depth is None else max_tree_depth + min_split = 20 if min_split is None and min_bucket is None else min_split + min_bucket = min_split // 3 if not min_bucket else min_bucket + min_split = min_bucket * 3 if not min_split else min_split + num_bins = 100 if num_bins is None else num_bins + learning_rate = 0.01 if learning_rate is None else learning_rate + if predict_dt_prob not in ["response", "prob", None]: + plpy.error(""" + invalid choice: '{predict_dt_prob}' (choose from 'response', 'prob') + """.format(predict_dt_prob= predict_dt_prob)) + predict_dt_prob = "response" if predict_dt_prob is None else predict_dt_prob + + null_handling_dict = extract_keyvalue_params( + null_handling_params, + dict(max_surrogates=int, null_as_category=bool), + dict(max_surrogates=0, null_as_category=False)) + max_n_surr = null_handling_dict['max_surrogates'] + null_as_category = null_handling_dict['null_as_category'] + null_proxy = "__NULL__" if null_as_category else None + if null_as_category: + # can't have two ways of handling tuples with NULL values + max_n_surr = 0 + _assert(max_n_surr >= 0, + "Maximum number of surrogates ({0}) should be non-negative". + format(max_n_surr)) + + # preprocess arguments + # expand "*" syntax and exclude some features + features = _get_features_to_use(schema_madlib, + training_table_name, list_of_features, list_of_features_to_exclude, + id_col_name, '1', dependent_variable, None) + + split_criterion = 'mse' + if num_random_features is None: + n_all_features = len(features) + num_random_features = int(sqrt(n_all_features) if is_classification else ceil(float(n_all_features) / 3)) + + _assert(0 < num_random_features <= len(features), + "GBDT error: Number of features to be selected " + "is more than the actual number of features.") + + all_cols_types = dict([(f, get_expr_type(f, training_table_name)) + for f in features]) + cat_features, ordered_cat_features, boolean_cats, con_features = \ + _classify_features(all_cols_types, features) + + + filter_null = _get_filter_str(dependent_variable, None) + + n_rows = plpy.execute("SELECT count(*) FROM {0} WHERE {1}".format(training_table_name, filter_null))[0]['count'] + dep_n_levels = 1 + + bins = _get_bins(schema_madlib, training_table_name, + cat_features, ordered_cat_features, + con_features, num_bins, dependent_variable, + boolean_cats, n_rows, False, + dep_n_levels, filter_null, null_proxy) + bins['grp_key_cat'] = [''] + cat_features = bins['cat_features'] + + tree = tree_fit(schema_madlib, bins, training_table_name, cat_features, con_features, + boolean_cats, num_bins,weights, + dependent_variable, min_split, min_bucket, + max_tree_depth, filter_null, dep_n_levels, + split_criterion, + num_random_features, + max_n_surr,null_proxy) + create_model_table(schema_madlib, output_table_name,tree, bins, 0) + + cat_features_str, con_features_str = get_feature_str(schema_madlib, + training_table_name, + cat_features, + con_features, + "m.cat_levels_in_text", "m.cat_n_levels", + null_proxy) + + id, y_pred = tree_gradient_predict(schema_madlib,output_table_name,0, id_col_name, cat_features_str, + con_features_str,is_classification,predict_dt_prob, training_table_name) + + y_res = plpy.execute("""SELECT {dependent_variable} AS y FROM {training_table_name}""".format(**locals())) + y = [] + for row in y_res: + y.append(row['y']) + + for sample_id in range(1, num_trees): + gradient = _get_gradient(y , y_pred, id, sample_id, training_table_name) + + tree = tree_fit (schema_madlib, bins, training_table_name, cat_features, con_features, + boolean_cats, num_bins, weights, + 'gradient', min_split, min_bucket, + max_tree_depth, filter_null, dep_n_levels, + split_criterion, + num_random_features, + max_n_surr,null_proxy) + + create_model_table(schema_madlib, output_table_name,tree, bins, sample_id) + id, y_temp = tree_gradient_predict(schema_madlib,output_table_name,sample_id, id_col_name, cat_features_str, + con_features_str,is_classification, predict_dt_prob,training_table_name) + y_pred -= np.multiply(learning_rate, y_temp) + + plpy.execute("""ALTER TABLE {training_table_name} DROP COLUMN IF EXISTS gradient CASCADE + """.format(training_table_name=training_table_name)) + + create_summary_table(output_table_name, null_proxy, bins['cat_features'], + bins['con_features'], learning_rate, is_classification, predict_dt_prob, + num_trees, training_table_name) + + + +def gbdt_predict(schema_madlib, test_table_name, model_table_name, output_table_name, id_col_name, **kwargs): + + num_tree = plpy.execute("""SELECT COUNT(*) AS count FROM {model_table_name}""".format(**locals()))[0]['count'] + if num_tree == 0: + plpy.error("The GBDT-method has no trees") + + + elements = plpy.execute("""SELECT * FROM {model_table_name}_summary""".format(**locals()))[0] + cat_features = [] if not elements['cat_features'] else elements['cat_features'].split(',') + con_features = [] if not elements['con_features'] else elements['con_features'].split(',') + null_proxy = elements['null_proxy'] + learning_rate = elements['learning_rate'] + is_classification = elements['is_classification'] + predict_dt_prob = elements['predict_dt_prob'] + + cat_features_str, con_features_str = get_feature_str( + schema_madlib, test_table_name, cat_features, con_features, "m.cat_levels_in_text", "m.cat_n_levels", null_proxy) + + id, y_pred = tree_gradient_predict(schema_madlib,model_table_name ,0, id_col_name, cat_features_str, + con_features_str,is_classification,predict_dt_prob,test_table_name) + + for i in range(1, num_tree): + _, gradient = tree_gradient_predict(schema_madlib,model_table_name , i, id_col_name, cat_features_str, + con_features_str,is_classification,predict_dt_prob,test_table_name) + y_pred -= np.multiply(learning_rate, gradient) + if is_classification: + y_pred = np.around(y_pred) + create_test_table(output_table_name, id, y_pred.tolist()) \ No newline at end of file diff --git a/src/ports/postgres/modules/gbdt/gbdt.sql_in b/src/ports/postgres/modules/gbdt/gbdt.sql_in new file mode 100644 index 0000000..70da99c --- /dev/null +++ b/src/ports/postgres/modules/gbdt/gbdt.sql_in @@ -0,0 +1,332 @@ +/* + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + */ + +m4_include(`SQLCommon.m4') + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER, + min_bucket INTEGER, + num_bins INTEGER, + null_handling_params TEXT, + is_classification BOOLEAN, + predict_dt_prob TEXT, + learning_rate DOUBLE PRECISION, + verbose BOOLEAN, + sample_ratio DOUBLE PRECISION +) +RETURNS VOID AS $$ +PythonFunction(gbdt, gbdt, gbdt_fit) +$$ LANGUAGE plpythonu VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + + + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER, + min_bucket INTEGER, + num_bins INTEGER, + null_handling_params TEXT, + is_classification BOOLEAN, + predict_dt_prob TEXT, + learning_rate DOUBLE PRECISION, + verbose BOOLEAN +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, $17, $18, NULL); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER, + min_bucket INTEGER, + num_bins INTEGER, + null_handling_params TEXT, + is_classification BOOLEAN, + predict_dt_prob TEXT, + learning_rate DOUBLE PRECISION +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, $17, FALSE::BOOLEAN); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER, + min_bucket INTEGER, + num_bins INTEGER, + null_handling_params TEXT, + is_classification BOOLEAN, + predict_dt_prob TEXT +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, NULL); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER, + min_bucket INTEGER, + num_bins INTEGER, + null_handling_params TEXT, + is_classification BOOLEAN +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, NULL); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER, + min_bucket INTEGER, + num_bins INTEGER, + null_handling_params TEXT +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, FALSE); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER, + min_bucket INTEGER, + num_bins INTEGER +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, NULL::TEXT); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER, + min_bucket INTEGER +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, 100::INTEGER); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER, + min_split INTEGER +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, ($11/3)::INTEGER); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER, + max_tree_depth INTEGER +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + 20::INTEGER); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER, + num_random_features INTEGER +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, $9, 1::INTEGER); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT, + num_trees INTEGER +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, $8, NULL::INTEGER); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT, + weights TEXT +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, $7, 30::INTEGER); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT, + list_of_features_to_exclude TEXT +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, $6, NULL::TEXT); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_train( + training_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT, + dependent_variable TEXT, + list_of_features TEXT +) +RETURNS VOID AS $$ + SELECT MADLIB_SCHEMA.gbdt_train($1, $2, $3, $4, $5, NULL::TEXT); +$$ LANGUAGE sql VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); + + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gbdt_predict( + test_table_name TEXT, + model_table_name TEXT, + output_table_name TEXT, + id_col_name TEXT +) RETURNS VOID AS $$ +PythonFunction(gbdt, gbdt, gbdt_predict) +$$ LANGUAGE plpythonu VOLATILE +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); \ No newline at end of file diff --git a/src/ports/postgres/modules/gbdt/test/gbdt.sql b/src/ports/postgres/modules/gbdt/test/gbdt.sql new file mode 100644 index 0000000..1bdbd28 --- /dev/null +++ b/src/ports/postgres/modules/gbdt/test/gbdt.sql @@ -0,0 +1,59 @@ +DROP TABLE IF EXISTS dt_golf CASCADE; +DROP TABLE IF EXISTS train_output,train_output_summary; +CREATE TABLE dt_golf ( + id integer NOT NULL, + "OUTLOOK" text, + temperature double precision, + humidity double precision, + "Cont_features" double precision[], + cat_features text[], + windy boolean, + class integer +) m4_ifdef(`__COLUMNS__', `WITH(ORIENTATION=COLUMN)', `'); + +INSERT INTO dt_golf (id,"OUTLOOK",temperature,humidity,"Cont_features",cat_features, windy,class) VALUES +(1, 'sunny', 85, 85,ARRAY[85, 85], ARRAY['a', 'b'], false, 0), +(2, 'sunny', 80, 90, ARRAY[80, 90], ARRAY['a', 'b'], true, 0), +(3, 'overcast', 83, 78, ARRAY[83, 78], ARRAY['a', 'b'], false, 1), +(4, 'rain', 70, NULL, ARRAY[70, 96], ARRAY['a', 'b'], false, 1), +(5, 'rain', 68, 80, ARRAY[68, 80], ARRAY['a', 'b'], false, 1), +(6, 'rain', NULL, 70, ARRAY[65, 70], ARRAY['a', 'b'], true, 0), +(7, 'overcast', 64, 65, ARRAY[64, 65], ARRAY['c', 'b'], NULL , 1), +(8, 'sunny', 72, 95, ARRAY[72, 95], ARRAY['a', 'b'], false, 0), +(9, 'sunny', 69, 70, ARRAY[69, 70], ARRAY['a', 'b'], false, 1), +(10, 'rain', 75, 80, ARRAY[75, 80], ARRAY['a', 'b'], false, 1), +(11, 'sunny', 75, 70, ARRAY[75, 70], ARRAY['a', 'd'], true, 1), +(12, 'overcast', 72, 90, ARRAY[72, 90], ARRAY['c', 'b'], NULL, 1), +(13, 'overcast', 81, 75, ARRAY[81, 75], ARRAY['a', 'b'], false, 1), +(15, NULL, 81, 75, ARRAY[81, 75], ARRAY['a', 'b'], false, 1), +(16, 'overcast', NULL, 75, ARRAY[81, 75], ARRAY['a', 'd'], false,1), +(14, 'rain', 71, 80, ARRAY[71, 80], ARRAY['c', 'b'], true, 0); + +------------------------------------------------------------------------- +-- classification without grouping + select madlib.gbdt_train('dt_golf', -- source table + 'train_output', -- output model table + 'id' , -- id column + 'class', -- response + 'windy, "Cont_features"[1]', -- features + NULL, -- exclude columns + NULL, + NULL, -- no grouping + 30, -- num of trees + 2, -- num of random features + TRUE, -- importance + 1, -- num_permutations + 10, -- max depth + 1, -- min split + 1, -- min bucket + 8, -- number of bins per continuous variable + 'max_surrogates=0', + 'response', + 0.001, + FALSE, + 0.5 +); + +CREATE TABLE dt_golf2 as SELECT * from dt_golf; +DROP TABLE IF EXISTS test_output; +select madlib.gbdt_predict('dt_golf2','train_output','test_output','id'); diff --git a/src/ports/postgres/modules/glm/glm.sql_in b/src/ports/postgres/modules/glm/glm.sql_in index 377dfea..4ebc6eb 100644 --- a/src/ports/postgres/modules/glm/glm.sql_in +++ b/src/ports/postgres/modules/glm/glm.sql_in @@ -649,7 +649,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_merge_states( state2 MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_merge_states' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -658,7 +658,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_final( state MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -671,7 +671,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_poisson_log_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_poisson_log_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -699,7 +699,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_poisson_identity_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_poisson_identity_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -727,7 +727,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_poisson_sqrt_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_poisson_sqrt_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -755,7 +755,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gaussian_identity_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_gaussian_identity_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -783,7 +783,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gaussian_log_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_gaussian_log_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -811,7 +811,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gaussian_inverse_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_gaussian_inverse_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -838,7 +838,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gamma_log_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_gamma_log_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -866,7 +866,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gamma_inverse_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_gamma_inverse_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -893,7 +893,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_gamma_identity_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_gamma_identity_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -921,7 +921,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_binomial_probit_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_binomial_probit_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -949,7 +949,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_inverse_gaussian_identity_transit MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_inverse_gaussian_identity_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -961,7 +961,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_binomial_logit_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_binomial_logit_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -986,7 +986,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_result_z_stats( /*+ state */ MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.__glm_result_type AS 'MODULE_PATHNAME', 'glm_result_z_stats' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1014,7 +1014,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_inverse_gaussian_log_transition( MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_inverse_gaussian_log_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1042,7 +1042,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_inverse_gaussian_inverse_transiti MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_inverse_gaussian_inverse_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1070,7 +1070,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_inverse_gaussian_sqr_inverse_tran MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'glm_inverse_gaussian_sqr_inverse_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1095,7 +1095,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_result_z_stats( /*+ state */ MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.__glm_result_type AS 'MODULE_PATHNAME', 'glm_result_z_stats' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1104,7 +1104,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_result_t_stats( /*+ state */ MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.__glm_result_type AS 'MODULE_PATHNAME', 'glm_result_t_stats' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1114,7 +1114,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__glm_loglik_diff( /*+ state2 */ MADLIB_SCHEMA.bytea8) RETURNS double precision AS 'MODULE_PATHNAME', 'glm_loglik_diff' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1212,7 +1212,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.glm_predict( link TEXT ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'glm_predict' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1236,7 +1236,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.glm_predict_binomial( link TEXT ) RETURNS BOOLEAN AS 'MODULE_PATHNAME', 'glm_predict_binomial' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1260,7 +1260,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.glm_predict_poisson( link TEXT ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'glm_predict_poisson' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/glm/multiresponseglm.sql_in b/src/ports/postgres/modules/glm/multiresponseglm.sql_in index c810305..46de254 100644 --- a/src/ports/postgres/modules/glm/multiresponseglm.sql_in +++ b/src/ports/postgres/modules/glm/multiresponseglm.sql_in @@ -556,7 +556,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_merge_states( state2 MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'multi_response_glm_merge_states' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -565,7 +565,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_final( state MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'multi_response_glm_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -579,7 +579,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_logit_transition( smallint) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'multi_response_glm_multinom_logit_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -605,7 +605,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_result( /*+ state */ MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.__multinom_result_type AS 'MODULE_PATHNAME', 'multi_response_glm_result_z_stats' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -615,7 +615,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__multinom_loglik_diff( /*+ state2 */ MADLIB_SCHEMA.bytea8) RETURNS double precision AS 'MODULE_PATHNAME', 'multi_response_glm_loglik_diff' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ diff --git a/src/ports/postgres/modules/glm/ordinal.sql_in b/src/ports/postgres/modules/glm/ordinal.sql_in index 7cda294..366f69c 100644 --- a/src/ports/postgres/modules/glm/ordinal.sql_in +++ b/src/ports/postgres/modules/glm/ordinal.sql_in @@ -492,7 +492,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_merge_states( state2 MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'ordinal_merge_states' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -501,7 +501,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_final( state MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'ordinal_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -514,7 +514,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_logit_transition( smallint) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'ordinal_logit_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -544,7 +544,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_probit_transition( smallint) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'ordinal_probit_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -570,7 +570,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_result( /*+ state */ MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.__ordinal_result_type AS 'MODULE_PATHNAME', 'ordinal_result' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -580,7 +580,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__ordinal_loglik_diff( /*+ state2 */ MADLIB_SCHEMA.bytea8) RETURNS double precision AS 'MODULE_PATHNAME', 'ordinal_loglik_diff' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ diff --git a/src/ports/postgres/modules/glm/test/multinom.sql_in b/src/ports/postgres/modules/glm/test/multinom.sql_in index 936aa08..9ca1a1f 100644 --- a/src/ports/postgres/modules/glm/test/multinom.sql_in +++ b/src/ports/postgres/modules/glm/test/multinom.sql_in @@ -39,7 +39,8 @@ CREATE TABLE multinom_test ( feat1 INTEGER, feat2 INTEGER, cat INTEGER, - g CHAR + g CHAR, + i serial ); INSERT INTO multinom_test(feat1, feat2, cat, g) VALUES @@ -274,7 +275,7 @@ WHERE category = '1'; -- for prediction -- adding ID column for prediction -ALTER TABLE multinom_test ADD COLUMN i SERIAL; +-- ALTER TABLE multinom_test ADD COLUMN i SERIAL; DROP TABLE IF EXISTS mglm_predict; SELECT multinom_predict( diff --git a/src/ports/postgres/modules/kmeans/kmeans.sql_in b/src/ports/postgres/modules/kmeans/kmeans.sql_in index 9e36a92..afa9a4d 100644 --- a/src/ports/postgres/modules/kmeans/kmeans.sql_in +++ b/src/ports/postgres/modules/kmeans/kmeans.sql_in @@ -996,7 +996,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_kmeans_args( ) RETURNS VOID VOLATILE CALLED ON NULL INPUT -LANGUAGE c +LANGUAGE c NOT FENCED AS 'MODULE_PATHNAME', 'exec_sql_using' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1202,22 +1202,22 @@ BEGIN -- Handle PG11 pg_proc table changes IF (SELECT MADLIB_SCHEMA.is_pg_major_version_less_than(11) = TRUE) THEN - IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = TRUE + IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = 't' FROM pg_proc WHERE oid = proc_fn_dist) THEN RAISE EXCEPTION 'Kmeans error: Distance function has wrong signature or is not a simple function.'; END IF; proc_agg_centroid := agg_centroid || '(DOUBLE PRECISION[])'; - IF (SELECT prorettype != 'DOUBLE PRECISION[]'::regtype OR proisagg = FALSE + IF (SELECT prorettype != 'DOUBLE PRECISION[]'::regtype OR proisagg != 't' FROM pg_proc WHERE oid = proc_agg_centroid) THEN RAISE EXCEPTION 'Kmeans error: Mean aggregate has wrong signature or is not an aggregate.'; END IF; ELSE - IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR prokind = 'a' + IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = 't' FROM pg_proc WHERE oid = proc_fn_dist) THEN RAISE EXCEPTION 'Kmeans error: Distance function has wrong signature or is not a simple function.'; END IF; proc_agg_centroid := agg_centroid || '(DOUBLE PRECISION[])'; - IF (SELECT prorettype != 'DOUBLE PRECISION[]'::regtype OR prokind != 'a' + IF (SELECT prorettype != 'DOUBLE PRECISION[]'::regtype OR proisagg != 't' FROM pg_proc WHERE oid = proc_agg_centroid) THEN RAISE EXCEPTION 'Kmeans error: Mean aggregate has wrong signature or is not an aggregate.'; END IF; @@ -1255,8 +1255,8 @@ BEGIN -- PostgreSQL 8.4: -- http://www.postgresql.org/docs/8.4/static/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN -- We therefore have to emulate. + DROP TABLE IF EXISTS pg_temp._madlib_kmeans_args; PERFORM MADLIB_SCHEMA.internal_execute_using_kmeans_args($sql$ - DROP TABLE IF EXISTS pg_temp._madlib_kmeans_args; CREATE TABLE pg_temp._madlib_kmeans_args AS SELECT $1 AS initial_centroids, @@ -1354,7 +1354,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_kmeanspp_seeding ) RETURNS VOID VOLATILE CALLED ON NULL INPUT -LANGUAGE c +LANGUAGE c NOT FENCED AS 'MODULE_PATHNAME', 'exec_sql_using' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1414,7 +1414,7 @@ BEGIN proc_fn_dist := fn_dist || '(DOUBLE PRECISION[], DOUBLE PRECISION[])'; IF (SELECT MADLIB_SCHEMA.is_pg_major_version_less_than(11) = TRUE) THEN - IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = TRUE + IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR prokind = 'a' FROM pg_proc WHERE oid = proc_fn_dist) THEN RAISE EXCEPTION 'Kmeans error: Distance function has wrong signature or is not a simple function.'; END IF; @@ -1463,8 +1463,8 @@ BEGIN -- PostgreSQL 8.4: -- http://www.postgresql.org/docs/8.4/static/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN -- We therefore have to emulate. + DROP TABLE IF EXISTS pg_temp._madlib_kmeanspp_args; PERFORM MADLIB_SCHEMA.internal_execute_using_kmeanspp_seeding_args($sql$ - DROP TABLE IF EXISTS pg_temp._madlib_kmeanspp_args; CREATE TEMPORARY TABLE _madlib_kmeanspp_args AS SELECT $1 AS k, @@ -1685,7 +1685,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_kmeans_random_se ) RETURNS VOID VOLATILE CALLED ON NULL INPUT -LANGUAGE c +LANGUAGE c NOT FENCED AS 'MODULE_PATHNAME', 'exec_sql_using' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1745,8 +1745,8 @@ BEGIN -- PostgreSQL 8.4: -- http://www.postgresql.org/docs/8.4/static/plpgsql-statements.html#PLPGSQL-STATEMENTS-EXECUTING-DYN -- We therefore have to emulate. + DROP TABLE IF EXISTS pg_temp._madlib_kmeans_random_args; PERFORM MADLIB_SCHEMA.internal_execute_using_kmeans_random_seeding_args($sql$ - DROP TABLE IF EXISTS pg_temp._madlib_kmeans_random_args; CREATE TEMPORARY TABLE _madlib_kmeans_random_args AS SELECT $1 AS k, $2 AS initial_centroids; $sql$, @@ -1924,7 +1924,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_kmeans_args( ) RETURNS MADLIB_SCHEMA.kmeans_result VOLATILE CALLED ON NULL INPUT -LANGUAGE c +LANGUAGE c NOT FENCED AS 'MODULE_PATHNAME', 'exec_sql_using' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -2060,7 +2060,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_execute_using_silhouette_args( ) RETURNS DOUBLE PRECISION STABLE CALLED ON NULL INPUT -LANGUAGE c +LANGUAGE c NOT FENCED AS 'MODULE_PATHNAME', 'exec_sql_using' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -2117,7 +2117,7 @@ BEGIN proc_fn_dist := fn_dist || '(DOUBLE PRECISION[], DOUBLE PRECISION[])'; IF (SELECT MADLIB_SCHEMA.is_pg_major_version_less_than(11) = TRUE) THEN - IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR proisagg = TRUE + IF (SELECT prorettype != 'DOUBLE PRECISION'::regtype OR prokind = 'a' FROM pg_proc WHERE oid = proc_fn_dist) THEN RAISE EXCEPTION 'Kmeans error: Distance function has wrong signature or is not a simple function.'; END IF; diff --git a/src/ports/postgres/modules/knn/knn.py_in b/src/ports/postgres/modules/knn/knn.py_in index eb2150e..1f5f055 100644 --- a/src/ports/postgres/modules/knn/knn.py_in +++ b/src/ports/postgres/modules/knn/knn.py_in @@ -122,7 +122,7 @@ def knn_validate_src(schema_madlib, point_source, point_column_name, point_id, if fn_dist: fn_dist = fn_dist.lower().strip() - profunc = ("proisagg = TRUE" + profunc = ("prokind = 'a'" if is_pg_major_version_less_than(schema_madlib, 11) else "prokind = 'a'") @@ -180,12 +180,13 @@ def build_kd_tree(schema_madlib, source_table, output_table, point_column_name, for curr_leaf in range(pow(2,curr_level)): clause = clauses[clause_counter] cutoff_sql = """ - SELECT percentile_disc(0.5) - WITHIN GROUP ( + SELECT {schema_madlib}.percentile_disc(array_agg(cutoff), 0.5) AS cutoff + FROM ( + SELECT {point_column_name}[{curr_feature}] AS cutoff + FROM {source_table} + WHERE {clause} and cutoff is not null ORDER BY ({point_column_name})[{curr_feature}] - ) AS cutoff - FROM {source_table} - WHERE {clause} + ) """.format(**locals()) cutoff = plpy.execute(cutoff_sql)[0]['cutoff'] diff --git a/src/ports/postgres/modules/knn/test/knn.sql_in b/src/ports/postgres/modules/knn/test/knn.sql_in index e4cd578..63a9883 100644 --- a/src/ports/postgres/modules/knn/test/knn.sql_in +++ b/src/ports/postgres/modules/knn/test/knn.sql_in @@ -143,18 +143,18 @@ SELECT assert(array_agg(prediction::numeric ORDER BY id)='{1,1,0.040872859187601 DROP TABLE if exists madlib_knn_result_classification; -SELECT knn('knn_train_data','data[1:1]||data[2:2]','id','label','knn_test_data','data[1:1]||data[2:2]','id','madlib_knn_result_classification',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True); +SELECT knn('knn_train_data','data[1,1]||data[2,2]','id','label','knn_test_data','data[1,1]||data[2,2]','id','madlib_knn_result_classification',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True); SELECT assert(array_agg(prediction::numeric ORDER BY id)='{1,1,0,1,0,0}', 'Wrong output in classification') FROM madlib_knn_result_classification; DROP TABLE if exists madlib_knn_result_regression; -SELECT knn('knn_train_data_reg','data[1:1]||data[2:2]','id','label','knn_test_data','data[1:1]||data[2:2]','id','madlib_knn_result_regression',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True); +SELECT knn('knn_train_data_reg','data[1,1]||data[2,2]','id','label','knn_test_data','data[1,1]||data[2,2]','id','madlib_knn_result_regression',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True); SELECT assert(array_agg(prediction::numeric ORDER BY id)='{1,1,0.0408728591876018,1,0,0}', 'Wrong output in regression') FROM madlib_knn_result_regression; DROP TABLE if exists madlib_knn_result_classification; -SELECT knn('knn_train_data_expr','ARRAY[data1,data2]','id','label','knn_test_data','data[1:1]||data[2:2]','id','madlib_knn_result_classification',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True); +SELECT knn('knn_train_data_expr','ARRAY[data1,data2]','id','label','knn_test_data','data[1,1]||data[2,2]','id','madlib_knn_result_classification',3,False,'MADLIB_SCHEMA.squared_dist_norm2', True); SELECT assert(array_agg(prediction::numeric ORDER BY id)='{1,1,0,1,0,0}', 'Wrong output in classification') FROM madlib_knn_result_classification; diff --git a/src/ports/postgres/modules/lda/lda.py_in b/src/ports/postgres/modules/lda/lda.py_in index 29c9d85..c34ced5 100644 --- a/src/ports/postgres/modules/lda/lda.py_in +++ b/src/ports/postgres/modules/lda/lda.py_in @@ -229,7 +229,7 @@ class LDATrainer: {schema_madlib}.__lda_count_topic_agg( words, counts, - doc_topic[{topic_num} + 1:array_upper(doc_topic, 1)], + doc_topic[{topic_num} + 1,array_upper(doc_topic, 1)], {voc_size}, {topic_num} ) AS model @@ -253,7 +253,7 @@ class LDATrainer: {schema_madlib}.__lda_count_topic_agg( words, counts, - doc_topic[{topic_num} + 1:array_upper(doc_topic, 1)], + doc_topic[{topic_num} + 1,array_upper(doc_topic, 1)], {voc_size}, {topic_num} ) AS model @@ -319,8 +319,8 @@ class LDATrainer: plpy.execute(""" INSERT INTO {output_data_table} SELECT - docid, wordcount, words, counts, doc_topic[1:{topic_num}] topic_count, - doc_topic[{topic_num} + 1:array_upper(doc_topic,1)] topic_assignment + docid, wordcount, words, counts, doc_topic[1,{topic_num}] topic_count, + doc_topic[{topic_num} + 1,array_upper(doc_topic,1)] topic_assignment FROM {work_table_final} """.format(output_data_table=self.output_data_table, @@ -448,8 +448,8 @@ class LDAPredictor: plpy.execute(""" INSERT INTO {doc_topic} SELECT - docid, wordcount, words, counts, doc_topic[1:{topic_num}] topic_count, - doc_topic[{topic_num} + 1:array_upper(doc_topic,1)] topic_assignment + docid, wordcount, words, counts, doc_topic[1,{topic_num}] topic_count, + doc_topic[{topic_num} + 1,array_upper(doc_topic,1)] topic_assignment FROM {work_table_out} """.format(doc_topic=self.doc_topic, topic_num=self.topic_num, @@ -1116,7 +1116,7 @@ def _validate_data_table(data_table, voc_size): rv = plpy.execute(""" SELECT - count(wordid) size, + count(wordid) AS size, min(wordid) min_wordid, max(wordid) max_wordid FROM diff --git a/src/ports/postgres/modules/lda/lda.sql_in b/src/ports/postgres/modules/lda/lda.sql_in index 814b0ae..a0910ca 100644 --- a/src/ports/postgres/modules/lda/lda.sql_in +++ b/src/ports/postgres/modules/lda/lda.sql_in @@ -1264,7 +1264,7 @@ MADLIB_SCHEMA.__lda_random_assign ) RETURNS INT4[] AS 'MODULE_PATHNAME', 'lda_random_assign' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1302,7 +1302,7 @@ MADLIB_SCHEMA.__lda_gibbs_sample ) RETURNS INT4[] AS 'MODULE_PATHNAME', 'lda_gibbs_sample' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1330,7 +1330,7 @@ MADLIB_SCHEMA.__lda_count_topic_sfunc ) RETURNS INT8[] AS 'MODULE_PATHNAME', 'lda_count_topic_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1348,7 +1348,7 @@ MADLIB_SCHEMA.__lda_count_topic_prefunc ) RETURNS INT8[] AS 'MODULE_PATHNAME', 'lda_count_topic_prefunc' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1436,7 +1436,7 @@ MADLIB_SCHEMA.__lda_perplexity_sfunc ) RETURNS INT8[] AS 'MODULE_PATHNAME', 'lda_perplexity_sfunc' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1453,7 +1453,7 @@ MADLIB_SCHEMA.__lda_perplexity_prefunc ) RETURNS INT8[] AS 'MODULE_PATHNAME', 'lda_perplexity_prefunc' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1468,7 +1468,7 @@ MADLIB_SCHEMA.__lda_perplexity_ffunc ) RETURNS FLOAT8 AS 'MODULE_PATHNAME', 'lda_perplexity_ffunc' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /* @@ -1526,7 +1526,7 @@ MADLIB_SCHEMA.__lda_check_count_ceiling ) RETURNS INT4[] AS 'MODULE_PATHNAME', 'lda_check_count_ceiling' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1543,7 +1543,7 @@ MADLIB_SCHEMA.__lda_util_unnest ) RETURNS SETOF INT4[] AS 'MODULE_PATHNAME', 'lda_unnest' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -1555,7 +1555,7 @@ MADLIB_SCHEMA.__lda_util_unnest_transpose ) RETURNS SETOF INT4[] AS 'MODULE_PATHNAME', 'lda_unnest_transpose' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1570,7 +1570,7 @@ MADLIB_SCHEMA.__lda_util_transpose ) RETURNS INT8[][] AS 'MODULE_PATHNAME', 'lda_transpose' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1587,7 +1587,7 @@ MADLIB_SCHEMA.__lda_util_norm_with_smoothing ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'l1_norm_with_smoothing' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1694,6 +1694,6 @@ MADLIB_SCHEMA.lda_parse_model ) RETURNS MADLIB_SCHEMA._pivotalr_lda_model AS 'MODULE_PATHNAME' -LANGUAGE c STRICT +LANGUAGE c STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/lda/test/lda.sql_in b/src/ports/postgres/modules/lda/test/lda.sql_in index 96d5f5f..0dc3b9a 100644 --- a/src/ports/postgres/modules/lda/test/lda.sql_in +++ b/src/ports/postgres/modules/lda/test/lda.sql_in @@ -276,7 +276,7 @@ CREATE OR REPLACE FUNCTION validate_lda_output() RETURNS integer AS $$ -- Loop through the output of lda_get_word_topic_count() and compare the results with the helper table word_topic_mapping FOR word_topic_count_row IN select * from word_topic_count ORDER BY wordid LOOP word_topic_count_topic_count := ARRAY[word_topic_count_row.topic_count]; - word_topic_mapping_topic_count := word_topic_mapping_array[word_topic_count_row.wordid+1:word_topic_count_row.wordid+1]; -- Here arrayX[i:i] means the ith 1d array of a 2d array + word_topic_mapping_topic_count := word_topic_mapping_array[word_topic_count_row.wordid+1,word_topic_count_row.wordid+1]; -- Here arrayX[i:i] means the ith 1d array of a 2d array IF (word_topic_mapping_topic_count != word_topic_count_topic_count) THEN RAISE EXCEPTION 'Topic assignment for wordid % does not match. word_topic_mapping: % word_topic_count: %',word_topic_count_row.wordid, word_topic_mapping_topic_count, word_topic_count_topic_count; diff --git a/src/ports/postgres/modules/linalg/linalg.sql_in b/src/ports/postgres/modules/linalg/linalg.sql_in index 3c74451..7570ae1 100644 --- a/src/ports/postgres/modules/linalg/linalg.sql_in +++ b/src/ports/postgres/modules/linalg/linalg.sql_in @@ -227,7 +227,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.norm1( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -242,7 +242,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.norm2( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -259,7 +259,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_inf_norm( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -278,7 +278,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_pnorm( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -295,7 +295,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_norm1( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -312,7 +312,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_norm2( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -331,7 +331,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.cosine_similarity( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -350,7 +350,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.squared_dist_norm2( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -368,7 +368,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_angle( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -387,7 +387,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_tanimoto( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -405,7 +405,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dist_jaccard( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /* @@ -424,7 +424,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._closest_column( dist_dn TEXT ) RETURNS MADLIB_SCHEMA.closest_column_result AS 'MODULE_PATHNAME', 'closest_column' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); @@ -508,7 +508,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._closest_columns( dist_dn TEXT ) RETURNS MADLIB_SCHEMA.closest_columns_result AS 'MODULE_PATHNAME', 'closest_columns' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.closest_columns( @@ -541,7 +541,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_vector_transition( x DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] LANGUAGE c -IMMUTABLE +IMMUTABLE NOT FENCED CALLED ON NULL INPUT AS 'MODULE_PATHNAME' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -552,7 +552,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_vector_merge( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -561,7 +561,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_vector_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -590,7 +590,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalized_avg_vector_transition( x DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] LANGUAGE c -IMMUTABLE +IMMUTABLE NOT FENCED CALLED ON NULL INPUT AS 'MODULE_PATHNAME' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -600,7 +600,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalized_avg_vector_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -632,7 +632,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.matrix_agg_transition( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -641,7 +641,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.matrix_agg_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -676,7 +676,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.matrix_column( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -690,7 +690,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.deconstruct_2d_array( in_array double precision[] ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'deconstruct_2d_array' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); ------------------------------ @@ -703,7 +703,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__deconstruct_lower_triangle( in_array double precision[] ) RETURNS SETOF record AS 'MODULE_PATHNAME', 'deconstruct_lower_triangle' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); ------------------- Created for In-Memory Iteration Controller ----------- @@ -716,7 +716,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_to_1d( in_array DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'array_to_1d' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -729,7 +729,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_to_2d( in_array DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'array_to_2d' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -744,7 +744,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.index_2d_array( ) RETURNS double precision[] AS 'MODULE_PATHNAME', 'get_row_from_2d_array' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -759,7 +759,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.get_row( ) RETURNS double precision[] AS 'MODULE_PATHNAME', 'get_row_from_2d_array' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -774,7 +774,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.get_col( ) RETURNS double precision[] AS 'MODULE_PATHNAME', 'get_col_from_2d_array' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); m4_changequote(, ) diff --git a/src/ports/postgres/modules/linalg/matrix_ops.py_in b/src/ports/postgres/modules/linalg/matrix_ops.py_in index bdbb502..b03d26d 100644 --- a/src/ports/postgres/modules/linalg/matrix_ops.py_in +++ b/src/ports/postgres/modules/linalg/matrix_ops.py_in @@ -2570,10 +2570,12 @@ def matrix_cholesky(schema_madlib, matrix_in, in_args, plpy.execute('DROP TABLE IF EXISTS %s' % matrix_p_temp) matrix_l_temp = "pg_temp." + unique_string() if is_output_sparse else matrix_l + dim_start = dim[0] + 1 + dim_end = dim[0] * 2 plpy.execute(""" CREATE TABLE {matrix_l_temp} AS SELECT row_id AS {out_args[row]}, - row_vec[{dim[0]} + 1 : {dim[0]} * 2] AS {out_args[val]} + row_vec[{dim_start}:{dim_end}] AS {out_args[val]} FROM {matrix_temp} """.format(**locals())) if is_output_sparse: @@ -2581,10 +2583,12 @@ def matrix_cholesky(schema_madlib, matrix_in, in_args, plpy.execute('DROP TABLE IF EXISTS %s' % matrix_l_temp) matrix_d_temp = "pg_temp." + unique_string() if is_output_sparse else matrix_d + dim_start = dim[0] * 2 + 1 + dim_end = dim[0] * 3 plpy.execute(""" CREATE TABLE {matrix_d_temp} AS SELECT row_id AS {out_args[row]}, - row_vec[{dim[0]} * 2 + 1: {dim[0]} * 3] AS {out_args[val]} + row_vec[{dim_start}:{dim_end}] AS {out_args[val]} FROM {matrix_temp} """.format(**locals())) if is_output_sparse: @@ -2673,10 +2677,12 @@ def matrix_qr(schema_madlib, matrix_in, in_args, matrix_out_prefix, out_args): plpy.execute('DROP TABLE IF EXISTS %s' % matrix_q_temp) matrix_r_temp = "pg_temp." + unique_string() if is_output_sparse else matrix_r + dim_start = dim[0] + 1 + dim_end = dim[0] + dim[1] plpy.execute(""" CREATE TABLE {matrix_r_temp} AS SELECT row_id as {out_args[row]}, - row_vec[{dim[0]} + 1 : {dim[0]} + {dim[1]}] AS {out_args[val]} + row_vec[{dim_start}:{dim_end}] AS {out_args[val]} FROM {matrix_temp} """.format(**locals())) if is_output_sparse: @@ -2829,22 +2835,28 @@ def matrix_lu(schema_madlib, matrix_in, in_args, row_vec[1:{dim[0]}] AS {out_args[val]} FROM {matrix_temp} WHERE row_id <= {dim[0]} """.format(**locals())) + dim_start = 2 * dim[0] + dim[1] + 1 + dim_end = 2 * dim[0] + 2 * dim[1] plpy.execute(""" CREATE TABLE {matrix_temp_names[q]} AS SELECT row_id AS {out_args[row]}, - row_vec[2 * {dim[0]} + {dim[1]} + 1:2 * {dim[0]} + 2 * {dim[1]}] AS {out_args[val]} + row_vec[{dim_start} : {dim_end}] AS {out_args[val]} FROM {matrix_temp} WHERE row_id <= {dim[1]} """.format(**locals())) + dim_start = dim[0] + 1 + dim_end = 2 * dim[0] plpy.execute(""" CREATE TABLE {matrix_temp_names[l]} AS SELECT row_id AS {out_args[row]}, - row_vec[{dim[0]} + 1 : 2 * {dim[0]}] AS {out_args[val]} + row_vec[{dim_start} : {dim_end}] AS {out_args[val]} FROM {matrix_temp} WHERE row_id <= {dim[0]} """.format(**locals())) + dim_start = 2 * dim[0] + 1 + dim_end = 2 * dim[0] + dim[1] plpy.execute(""" CREATE TABLE {matrix_temp_names[u]} AS SELECT row_id AS {out_args[row]}, - row_vec[2 * {dim[0]} + 1 : 2 * {dim[0]} + {dim[1]}] AS {out_args[val]} + row_vec[{dim_start} : {dim_end}] AS {out_args[val]} FROM {matrix_temp} WHERE row_id <= {dim[0]} """.format(**locals())) diff --git a/src/ports/postgres/modules/linalg/matrix_ops.sql_in b/src/ports/postgres/modules/linalg/matrix_ops.sql_in index 6e33f46..06eb944 100644 --- a/src/ports/postgres/modules/linalg/matrix_ops.sql_in +++ b/src/ports/postgres/modules/linalg/matrix_ops.sql_in @@ -3000,7 +3000,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_vec_mult_in_mem( ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'matrix_vec_mult_in_mem_2d' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /* @@ -3015,7 +3015,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_vec_mult_in_mem( ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'matrix_vec_mult_in_mem_1d' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -3031,7 +3031,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__row_fold( ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'row_fold' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -3045,7 +3045,7 @@ MADLIB_SCHEMA.__bernoulli_vector ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'bernoulli_vector' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -3068,7 +3068,7 @@ MADLIB_SCHEMA.__uniform_vector ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'uniform_vector' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -3093,7 +3093,7 @@ MADLIB_SCHEMA.__normal_vector ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'normal_vector' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -3116,7 +3116,7 @@ MADLIB_SCHEMA.__rand_vector ) RETURNS INTEGER[] AS 'MODULE_PATHNAME', 'rand_vector' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -3128,7 +3128,7 @@ MADLIB_SCHEMA.__rand_block ) RETURNS INTEGER[] AS 'MODULE_PATHNAME', 'rand_block' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -3149,7 +3149,7 @@ MADLIB_SCHEMA.__matrix_row_split ) RETURNS SETOF FLOAT8[] AS 'MODULE_PATHNAME', 'row_split' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -3162,7 +3162,7 @@ MADLIB_SCHEMA.__matrix_densify_sfunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'matrix_densify_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS @@ -3222,7 +3222,7 @@ MADLIB_SCHEMA.__matrix_blockize_sfunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'matrix_blockize_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS @@ -3255,7 +3255,7 @@ MADLIB_SCHEMA.__matrix_unblockize_sfunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'matrix_unblockize_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS @@ -3287,7 +3287,7 @@ MADLIB_SCHEMA.matrix_mem_mult ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'matrix_mem_mult' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -3310,7 +3310,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); -- ) -- RETURNS FLOAT8[] -- AS 'MODULE_PATHNAME', 'matrix_vec_mem_mult' --- $$ LANGUAGE C; +-- $$ LANGUAGE C NOT FENCED; CREATE OR REPLACE FUNCTION @@ -3320,7 +3320,7 @@ MADLIB_SCHEMA.matrix_mem_trans ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'matrix_mem_trans' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -3332,7 +3332,7 @@ MADLIB_SCHEMA.__matrix_mem_sum_sfunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'matrix_mem_sum_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS @@ -3427,7 +3427,7 @@ MADLIB_SCHEMA.__matrix_unnest_block ) RETURNS SETOF FLOAT8[] AS 'MODULE_PATHNAME', 'unnest_block' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------- @@ -3439,7 +3439,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_compose_dense_transition( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_compose_dense_transition' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3453,7 +3453,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_compose_sparse_transition( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_compose_sparse_transition' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3463,7 +3463,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_compose_merge( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_compose_merge' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3472,7 +3472,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_inv_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_inv' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3481,7 +3481,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_pinv_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_pinv' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3490,7 +3490,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_eigen_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_eigen' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3499,7 +3499,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_cholesky_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_cholesky' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3508,7 +3508,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_qr_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_qr' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3517,7 +3517,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_rank_final( ) RETURNS INT8 LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_rank' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3526,7 +3526,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_lu_final( ) RETURNS DOUBLE PRECISION[] LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_lu' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); @@ -3535,7 +3535,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__matrix_nuclear_norm_final( ) RETURNS DOUBLE PRECISION LANGUAGE c IMMUTABLE -STRICT +STRICT NOT FENCED AS 'MODULE_PATHNAME', 'matrix_nuclear_norm' m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); diff --git a/src/ports/postgres/modules/linalg/pivotalr_arrayops.sql_in b/src/ports/postgres/modules/linalg/pivotalr_arrayops.sql_in index ff569d8..0f069ee 100644 --- a/src/ports/postgres/modules/linalg/pivotalr_arrayops.sql_in +++ b/src/ports/postgres/modules/linalg/pivotalr_arrayops.sql_in @@ -6,7 +6,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_crossprod_transition( right_arr DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', '__pivotalr_crossprod_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_crossprod_merge( @@ -14,7 +14,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_crossprod_merge( right_state DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', '__pivotalr_crossprod_merge' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.crossprod(DOUBLE PRECISION[], DOUBLE PRECISION[]); @@ -36,7 +36,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_crossprod_sym_transition( arr DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', '__pivotalr_crossprod_sym_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.crossprod_sym(DOUBLE PRECISION[]); @@ -57,14 +57,14 @@ CREATE AGGREGATE MADLIB_SCHEMA.crossprod_sym( -- arr DOUBLE PRECISION[] -- ) RETURNS DOUBLE PRECISION[] AS -- 'MODULE_PATHNAME', '__pivotalr_array_mean_transition' --- LANGUAGE C IMMUTABLE; +-- LANGUAGE C IMMUTABLE NOT FENCED; -- CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__pivotalr_array_sum_merge( -- left_state DOUBLE PRECISION[], -- right_state DOUBLE PRECISION[] -- ) RETURNS DOUBLE PRECISION[] AS -- 'MODULE_PATHNAME', '__pivotalr_array_mean_merge' --- LANGUAGE C IMMUTABLE; +-- LANGUAGE C IMMUTABLE NOT FENCED; -- CREATE AGGREGATE MADLIB_SCHEMA.pivotalr_array_sum( -- /* vector */ DOUBLE PRECISION[] diff --git a/src/ports/postgres/modules/linalg/svd.sql_in b/src/ports/postgres/modules/linalg/svd.sql_in index 50313c5..a7115ac 100644 --- a/src/ports/postgres/modules/linalg/svd.sql_in +++ b/src/ports/postgres/modules/linalg/svd.sql_in @@ -677,7 +677,7 @@ MADLIB_SCHEMA.__svd_unit_vector -- RETURNS MADLIB_SCHEMA.__svd_unit_vector_result RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'svd_unit_vector' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP TYPE IF EXISTS MADLIB_SCHEMA.__svd_lanczos_result CASCADE; @@ -705,7 +705,7 @@ MADLIB_SCHEMA.__svd_lanczos_sfunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svd_lanczos_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------------------------------------------- @@ -723,7 +723,7 @@ MADLIB_SCHEMA.__svd_block_lanczos_sfunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svd_block_lanczos_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------------------------------------------- @@ -741,7 +741,7 @@ MADLIB_SCHEMA.__svd_sparse_lanczos_sfunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svd_sparse_lanczos_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -752,7 +752,7 @@ MADLIB_SCHEMA.__svd_lanczos_prefunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svd_lanczos_prefunc' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------------------------------------------- @@ -860,7 +860,7 @@ MADLIB_SCHEMA.__svd_lanczos_pvec ) RETURNS MADLIB_SCHEMA.__svd_lanczos_result AS 'MODULE_PATHNAME', 'svd_lanczos_pvec' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -872,7 +872,7 @@ MADLIB_SCHEMA.__svd_lanczos_qvec ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svd_lanczos_qvec' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------------------------------------------- @@ -887,7 +887,7 @@ MADLIB_SCHEMA.__svd_gram_schmidt_orthogonalize_sfunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svd_gram_schmidt_orthogonalize_sfunc' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION @@ -898,7 +898,7 @@ MADLIB_SCHEMA.__svd_gram_schmidt_orthogonalize_prefunc ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svd_gram_schmidt_orthogonalize_prefunc' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- This function will also do the normalization @@ -909,7 +909,7 @@ MADLIB_SCHEMA.__svd_gram_schmidt_orthogonalize_ffunc ) RETURNS MADLIB_SCHEMA.__svd_lanczos_result AS 'MODULE_PATHNAME', 'svd_gram_schmidt_orthogonalize_ffunc' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS @@ -1003,7 +1003,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__svd_decompose_bidiag( ) RETURNS MADLIB_SCHEMA.__svd_bidiagonal_matrix_result AS 'MODULE_PATHNAME', 'svd_decompose_bidiag' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ----------------------------------------------------------------------- @@ -1022,7 +1022,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__svd_vec_mult_matrix( ) RETURNS FLOAT8[] AS 'MODULE_PATHNAME', 'svd_vec_mult_matrix' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP TYPE IF EXISTS MADLIB_SCHEMA.__svd_vec_mat_mult_result CASCADE; @@ -1042,7 +1042,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__svd_vec_trans_mult_matrix_internal( ) RETURNS SETOF MADLIB_SCHEMA.__svd_vec_mat_mult_result AS 'MODULE_PATHNAME', 'svd_vec_trans_mult_matrix' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --Multiplication of P/Q vectors with Left/Right Singular Matrix of B diff --git a/src/ports/postgres/modules/linalg/test/linalg.sql_in b/src/ports/postgres/modules/linalg/test/linalg.sql_in index 4777d8d..b20bd44 100644 --- a/src/ports/postgres/modules/linalg/test/linalg.sql_in +++ b/src/ports/postgres/modules/linalg/test/linalg.sql_in @@ -127,7 +127,7 @@ SELECT assert( 'Incorrect closest column.' ) FROM ( SELECT - squared_dist_norm2(matrix[column_id + lb:column_id + lb], x) AS dist1, + squared_dist_norm2(matrix[column_id + lb,column_id + lb], x) AS dist1, distance AS dist2, least AS dist3, * diff --git a/src/ports/postgres/modules/linalg/test/matrix_ops.sql_in b/src/ports/postgres/modules/linalg/test/matrix_ops.sql_in index 8a7eb95..234373f 100644 --- a/src/ports/postgres/modules/linalg/test/matrix_ops.sql_in +++ b/src/ports/postgres/modules/linalg/test/matrix_ops.sql_in @@ -216,3 +216,22 @@ FROM ( SELECT array_to_1d(__matrix_sparse_inverse(3, 3, row_id - 1, col_id - 1, val)) as result FROM random_sparse_matrix ) q; + +CREATE TABLE "mat_A" ( + row_id integer, + row_vec integer[] +); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (1, '{9,6,5,8,5,6,6,3,10,8}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (2, '{8,2,2,6,6,10,2,1,9,9}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (3, '{3,9,9,9,8,6,3,9,5,6}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (4, '{6,4,2,2,2,7,8,8,0,7}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (5, '{6,8,9,9,4,6,9,5,7,7}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (6, '{4,10,7,3,9,5,9,2,3,4}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (7, '{8,10,7,10,1,9,7,9,8,7}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (8, '{7,4,5,6,2,8,1,1,4,8}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (9, '{8,8,8,5,2,6,9,1,8,3}'); +INSERT INTO "mat_A" (row_id, row_vec) VALUES (10, '{4,6,3,2,6,4,1,2,3,8}'); + +SELECT madlib.matrix_cholesky('"mat_A"', 'row=row_id, val=row_vec', 'matrix_out_prefix1'); +SELECT madlib.matrix_qr('"mat_A"', 'row=row_id, val=row_vec', 'matrix_out_prefix2'); +SELECT madlib.matrix_lu('"mat_A"', 'row=row_id, val=row_vec', 'matrix_out_prefix3'); diff --git a/src/ports/postgres/modules/linear_systems/dense_linear_systems.sql_in b/src/ports/postgres/modules/linear_systems/dense_linear_systems.sql_in index 15886f7..b1f2777 100644 --- a/src/ports/postgres/modules/linear_systems/dense_linear_systems.sql_in +++ b/src/ports/postgres/modules/linear_systems/dense_linear_systems.sql_in @@ -249,7 +249,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_residual_norm_transition( x DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_residual_norm_merge_states( @@ -257,14 +257,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_residual_norm_merge_states( state2 MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_residual_norm_final( state MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.residual_norm_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -315,7 +315,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_direct_linear_system_transition( algorithm INTEGER) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_direct_linear_system_merge_states( @@ -323,14 +323,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_direct_linear_system_merge_states state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.dense_direct_linear_system_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.dense_linear_solver_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** diff --git a/src/ports/postgres/modules/linear_systems/sparse_linear_systems.sql_in b/src/ports/postgres/modules/linear_systems/sparse_linear_systems.sql_in index a61a48e..a9ef39a 100644 --- a/src/ports/postgres/modules/linear_systems/sparse_linear_systems.sql_in +++ b/src/ports/postgres/modules/linear_systems/sparse_linear_systems.sql_in @@ -362,7 +362,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_inmem_iterative_linear_system_tr termToler DOUBLE PRECISION) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_inmem_iterative_linear_system_merge_states( @@ -370,14 +370,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_inmem_iterative_linear_system_me state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_inmem_iterative_linear_system_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.sparse_linear_solver_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -456,7 +456,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_direct_linear_system_transition( algorithm INTEGER) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_direct_linear_system_merge_states( @@ -464,7 +464,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_direct_linear_system_merge_state state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -472,7 +472,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.sparse_direct_linear_system_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.sparse_linear_solver_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/pca/pca.py_in b/src/ports/postgres/modules/pca/pca.py_in index cab8e7f..3882f24 100644 --- a/src/ports/postgres/modules/pca/pca.py_in +++ b/src/ports/postgres/modules/pca/pca.py_in @@ -166,9 +166,9 @@ def pca_wrap(schema_madlib, source_table, pc_table, row_id, # declare variables whose values will be different for each group, if # grouping_cols is specified - grouping_where_clause = '' - sparse_where_condition = '' - select_grouping_cols = '' + grouping_where_clause = ' ' + sparse_where_condition = ' ' + select_grouping_cols = ' ' temp_table_columns = '' result_summary_table_temp = '' # For Dense matrix format only: diff --git a/src/ports/postgres/modules/pca/pca_project.py_in b/src/ports/postgres/modules/pca/pca_project.py_in index d205044..64e97fd 100644 --- a/src/ports/postgres/modules/pca/pca_project.py_in +++ b/src/ports/postgres/modules/pca/pca_project.py_in @@ -416,10 +416,10 @@ def pca_project_wrap(schema_madlib, source_table, pc_table, out_table, # declare variables whose values will be different for each group, if # grouping_cols is specified - grouping_where_clause = '' - sparse_where_condition = '' - select_grouping_cols = '' - grouping_cols_values = '' + grouping_where_clause = ' ' + sparse_where_condition = ' ' + select_grouping_cols = ' ' + grouping_cols_values = ' ' result_summary_table_temp = '' other_columns_in_pc_table = [col for col in get_cols(pc_table) if col not in grouping_cols_list] temp_pc_table_columns = ', '.join(other_columns_in_pc_table) diff --git a/src/ports/postgres/modules/prob/prob.sql_in b/src/ports/postgres/modules/prob/prob.sql_in index 5322c29..88dfe63 100644 --- a/src/ports/postgres/modules/prob/prob.sql_in +++ b/src/ports/postgres/modules/prob/prob.sql_in @@ -144,7 +144,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.bernoulli_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -162,9 +162,20 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.bernoulli_pmf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.bernoulli_pmf( + x DOUBLE PRECISION, + sp DOUBLE PRECISION +) RETURNS VOID AS $$ +DECLARE +BEGIN + RAISE EXCEPTION 'The first element must be INT'; +END +$$ LANGUAGE plpgsql IMMUTABLE STRICT +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); + /** * @brief Bernoulli quantile function * @@ -178,7 +189,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.bernoulli_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -198,7 +209,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.beta_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -218,7 +229,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.beta_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -238,7 +249,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.beta_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -259,7 +270,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.binomial_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -279,7 +290,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.binomial_pmf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -301,7 +312,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.binomial_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -322,7 +333,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.cauchy_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -342,7 +353,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.cauchy_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -362,7 +373,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.cauchy_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -380,7 +391,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi_squared_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -398,7 +409,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi_squared_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -416,7 +427,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi_squared_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -434,7 +445,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.exponential_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -452,7 +463,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.exponential_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -470,7 +481,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.exponential_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -491,7 +502,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.extreme_value_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -511,7 +522,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.extreme_value_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -531,7 +542,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.extreme_value_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -551,7 +562,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fisher_f_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -571,7 +582,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fisher_f_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -591,7 +602,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.fisher_f_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -612,7 +623,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gamma_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -632,7 +643,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gamma_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -652,7 +663,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gamma_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -669,7 +680,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.geometric_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -687,9 +698,20 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.geometric_pmf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.geometric_pmf( + x DOUBLE PRECISION, + sp DOUBLE PRECISION +) RETURNS VOID AS $$ +DECLARE +BEGIN + RAISE EXCEPTION 'The first element must be INT'; +END +$$ LANGUAGE plpgsql IMMUTABLE STRICT +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); + /** * @brief Geometric quantile function * @@ -707,7 +729,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.geometric_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -735,7 +757,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hypergeometric_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -760,7 +782,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hypergeometric_pmf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -786,7 +808,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hypergeometric_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -807,7 +829,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.inverse_gamma_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -827,7 +849,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.inverse_gamma_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -847,7 +869,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.inverse_gamma_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -865,7 +887,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.kolmogorov_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -885,7 +907,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.laplace_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -905,7 +927,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.laplace_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -925,7 +947,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.laplace_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -945,7 +967,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logistic_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -965,7 +987,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logistic_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -985,7 +1007,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logistic_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1006,7 +1028,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lognormal_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1026,7 +1048,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lognormal_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1046,7 +1068,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.lognormal_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1067,7 +1089,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.negative_binomial_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1088,7 +1110,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.negative_binomial_pmf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1111,7 +1133,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.negative_binomial_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1134,7 +1156,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_beta_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1157,7 +1179,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_beta_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1180,7 +1202,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_beta_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1201,7 +1223,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_chi_squared_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1221,7 +1243,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_chi_squared_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1241,7 +1263,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_chi_squared_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1264,7 +1286,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_f_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1286,7 +1308,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_f_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1308,7 +1330,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_f_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1329,7 +1351,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_t_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1349,7 +1371,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_t_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1369,7 +1391,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.non_central_t_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1389,7 +1411,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_cdf( @@ -1428,7 +1450,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_pdf( @@ -1469,7 +1491,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normal_quantile( @@ -1511,7 +1533,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pareto_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1531,7 +1553,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pareto_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1551,7 +1573,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pareto_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1569,7 +1591,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1586,9 +1608,20 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_pmf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_pmf( + x DOUBLE PRECISION, + sp DOUBLE PRECISION +) RETURNS VOID AS $$ +DECLARE +BEGIN + RAISE EXCEPTION 'The first element must be INT'; +END +$$ LANGUAGE plpgsql IMMUTABLE STRICT +m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `'); + /** * @brief Poisson quantile function * @@ -1605,7 +1638,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1623,7 +1656,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rayleigh_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1640,7 +1673,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rayleigh_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1657,7 +1690,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rayleigh_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1675,7 +1708,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.students_t_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1693,7 +1726,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.students_t_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1711,7 +1744,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.students_t_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1733,7 +1766,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.triangular_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1754,7 +1787,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.triangular_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1775,7 +1808,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.triangular_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1795,7 +1828,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.uniform_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1814,7 +1847,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.uniform_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1833,7 +1866,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.uniform_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1854,7 +1887,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weibull_cdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1874,7 +1907,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weibull_pdf( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1894,5 +1927,5 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weibull_quantile( ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in index 69a54f3..2cffe99 100644 --- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in +++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in @@ -398,7 +398,7 @@ def _build_tree(schema_madlib, is_classification, split_criterion, FROM {1} """.format(grouping_array_str, cp_table) grp_cp_values = plpy.execute(sql) - grp_key_to_cp = dict((row['grp_key'], row['cp_val']) for row in grp_cp_values) + grp_key_to_cp = dict((""if row['grp_key'] is None else row['grp_key'], row['cp_val']) for row in grp_cp_values) with MinWarning(msg_level): plpy.notice("Building tree for cross validation") @@ -2144,7 +2144,7 @@ def _xvalidate(schema_madlib, tree_states, training_table_name, output_table_nam validation_result = plpy.execute(validation_result_query) plpy.notice("finished validation_result_query, validation_result = " + str(list(validation_result))) - grp_key_to_best_cp = dict((row['grp_key'], row['cp']) for row in validation_result) + grp_key_to_best_cp = dict((""if row['grp_key'] is None else row['grp_key'], row['cp']) for row in validation_result) # 4) update tree_states to have the best cp cross-validated for tree in tree_states: diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in index 04f7b82..613d7e6 100644 --- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in +++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in @@ -1659,14 +1659,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_con_splits_transition( num_splits SMALLINT ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'dst_compute_con_splits_transition' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_con_splits_final( state MADLIB_SCHEMA.bytea8 ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'dst_compute_con_splits_final' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA._dst_compute_con_splits( @@ -1694,7 +1694,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_entropy_transition( num_dep_var integer -- constant for the state size ) RETURNS integer[] AS 'MODULE_PATHNAME', 'dst_compute_entropy_transition' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1703,7 +1703,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_entropy_merge( state2 integer[] ) RETURNS integer[] AS 'MODULE_PATHNAME', 'dst_compute_entropy_merge' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1711,7 +1711,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dst_compute_entropy_final( state integer[] ) RETURNS double precision AS 'MODULE_PATHNAME', 'dst_compute_entropy_final' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1738,7 +1738,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._map_catlevel_to_int( null_as_category BOOLEAN -- flag to check if NULL is treated as a separate category ) RETURNS INTEGER[] AS 'MODULE_PATHNAME', 'map_catlevel_to_int' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -1749,7 +1749,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._initialize_decision_tree( max_n_surr SMALLINT ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'initialize_decision_tree' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1766,7 +1766,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_leaf_stats_transition( weights_as_rows BOOLEAN ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'compute_leaf_stats_transition' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1775,7 +1775,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_leaf_stats_merge( state2 MADLIB_SCHEMA.BYTEA8 ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'compute_leaf_stats_merge' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- One step in the iteration @@ -1827,7 +1827,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dt_apply( num_random_features INTEGER ) RETURNS MADLIB_SCHEMA._tree_result_type AS 'MODULE_PATHNAME', 'dt_apply' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------------------------------------------------- @@ -1842,7 +1842,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_surr_stats_transition( dup_count INTEGER ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME', 'compute_surr_stats_transition' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA._compute_surr_stats( @@ -1874,7 +1874,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._dt_surr_apply( con_splits MADLIB_SCHEMA.BYTEA8 ) RETURNS MADLIB_SCHEMA.BYTEA8 AS 'MODULE_PATHNAME', 'dt_surr_apply' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------- @@ -1896,7 +1896,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._print_decision_tree( tree MADLIB_SCHEMA.BYTEA8 ) RETURNS MADLIB_SCHEMA._flattened_tree AS 'MODULE_PATHNAME', 'print_decision_tree' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------- @@ -1906,7 +1906,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_var_importance( n_con_features INTEGER ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'compute_variable_importance' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------- @@ -1916,7 +1916,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._predict_dt_response( con_features DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'predict_dt_response' -LANGUAGE C VOLATILE +LANGUAGE C VOLATILE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------- @@ -1926,7 +1926,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._predict_dt_prob( con_features DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'predict_dt_prob' -LANGUAGE C VOLATILE +LANGUAGE C VOLATILE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -2119,7 +2119,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_decision_tree( verbose BOOLEAN ) RETURNS TEXT AS 'MODULE_PATHNAME', 'display_decision_tree' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_decision_tree( @@ -2143,7 +2143,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_decision_tree_surrogate( cat_n_levels INTEGER[] ) RETURNS TEXT AS 'MODULE_PATHNAME', 'display_decision_tree_surrogate' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_text_decision_tree( @@ -2155,7 +2155,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._display_text_decision_tree( dependent_levels TEXT[] ) RETURNS TEXT AS 'MODULE_PATHNAME', 'display_text_tree' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -2422,7 +2422,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._prune_and_cplist( compute_cp_list BOOLEAN ) RETURNS MADLIB_SCHEMA._prune_result_type AS 'MODULE_PATHNAME', 'prune_and_cplist' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -2433,7 +2433,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._convert_to_rpart_format( n_cats INTEGER ) RETURNS DOUBLE PRECISION[][] AS 'MODULE_PATHNAME', 'convert_to_rpart_format' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -2444,7 +2444,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._get_split_thresholds( n_cats integer ) RETURNS double precision[][] AS 'MODULE_PATHNAME', 'get_split_thresholds' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------- diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in index fae91bd..209b86a 100644 --- a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in +++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in @@ -333,7 +333,7 @@ def forest_train( if grouping_cols is None: sql_grp_key_to_grp_cols = """ CREATE TABLE {grp_key_to_grp_cols} AS - SELECT ''::text AS grp_key, 1 AS gid + SELECT ' '::text AS grp_key, 1 AS gid """.format(**locals()) plpy.notice("sql_grp_key_to_grp_cols:\n" + sql_grp_key_to_grp_cols) plpy.execute(sql_grp_key_to_grp_cols) @@ -504,13 +504,13 @@ def forest_train( is_classification, split_criterion, True, num_random_features, max_n_surr, null_proxy) - tree['grp_key'] = '' + tree['grp_key'] = ' ' if importance: tree.update(_compute_var_importance( schema_madlib, tree, len(cat_features), len(con_features))) tree_states = [tree] - tree_terminated = {'': tree['finished']} + tree_terminated = {' ': tree['finished']} else: tree_states = _tree_train_grps_using_bins( schema_madlib, bins, src_view, cat_features, con_features, @@ -837,7 +837,7 @@ def get_tree(schema_madlib, model_table, gid, sample_id, SELECT {0}._display_decision_tree( $1, $2, $3, $4, $5, $6, '{1}', {2} ) as display_tree - """.format(schema_madlib, "", verbose) + """.format(schema_madlib, " ", verbose) else: sql_display = """ SELECT {0}._display_text_decision_tree( diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in index 888388c..ae79129 100644 --- a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in +++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in @@ -1562,7 +1562,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._get_bin_value_by_index( bin_index INTEGER ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'get_bin_value_by_index' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `'); ----------------------------------------------------------- @@ -1572,7 +1572,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._get_bin_index_by_value( feature_index INTEGER ) RETURNS integer AS 'MODULE_PATHNAME', 'get_bin_index_by_value' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `'); ------------------------------------------------------------ @@ -1581,7 +1581,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._get_bin_indices_by_values( con_splits MADLIB_SCHEMA.bytea8 ) RETURNS integer[] AS 'MODULE_PATHNAME', 'get_bin_indices_by_values' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `'); ------------------------------------------------------------ @@ -2058,7 +2058,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._convert_to_random_forest_format( model MADLIB_SCHEMA.bytea8 ) RETURNS DOUBLE PRECISION[][] AS 'MODULE_PATHNAME', 'convert_to_random_forest_format' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- Helper functions for variable importance @@ -2073,7 +2073,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._rf_cat_imp_score( cat_feature_distributions DOUBLE PRECISION[][] ) RETURNS DOUBLE PRECISION[][] AS 'MODULE_PATHNAME', 'rf_cat_imp_score' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._rf_con_imp_score( @@ -2087,7 +2087,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._rf_con_imp_score( con_index_distrbutions DOUBLE PRECISION[][] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'rf_con_imp_score' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -2096,5 +2096,5 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.normalize_sum_array( target_sum DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'normalize_sum_array' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/regress/clustered_variance.sql_in b/src/ports/postgres/modules/regress/clustered_variance.sql_in index f05630d..2f7345b 100644 --- a/src/ports/postgres/modules/regress/clustered_variance.sql_in +++ b/src/ports/postgres/modules/regress/clustered_variance.sql_in @@ -514,7 +514,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_lin_transition ( coef DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -524,7 +524,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_lin_merge ( state2 MADLIB_SCHEMA.bytea8 ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -533,7 +533,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_lin_final ( state MADLIB_SCHEMA.bytea8 ) RETURNS MADLIB_SCHEMA.__clustered_agg_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -568,7 +568,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_lin_compute_stats ( numRows INTEGER ) RETURNS MADLIB_SCHEMA.__clustered_lin_result AS 'MODULE_PATHNAME', 'clustered_lin_compute_stats' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -753,7 +753,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_log_transition ( coef DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -763,7 +763,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_log_merge ( state2 MADLIB_SCHEMA.bytea8 ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -772,7 +772,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_log_final ( state MADLIB_SCHEMA.bytea8 ) RETURNS MADLIB_SCHEMA.__clustered_agg_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -807,7 +807,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_log_compute_stats ( numRows INTEGER ) RETURNS MADLIB_SCHEMA.__clustered_log_result AS 'MODULE_PATHNAME', 'clustered_log_compute_stats' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -925,7 +925,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_mlog_transition ( ref_cat INTEGER ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -935,7 +935,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_mlog_merge ( state2 MADLIB_SCHEMA.bytea8 ) RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -944,7 +944,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_err_mlog_final ( state MADLIB_SCHEMA.bytea8 ) RETURNS MADLIB_SCHEMA.__clustered_agg_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -982,7 +982,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__clustered_mlog_compute_stats ( numRows INTEGER ) RETURNS MADLIB_SCHEMA.__clustered_mlog_result AS 'MODULE_PATHNAME', 'clustered_mlog_compute_stats' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- DEPRECATED NOTICE ----------------------------------------------------------- diff --git a/src/ports/postgres/modules/regress/linear.sql_in b/src/ports/postgres/modules/regress/linear.sql_in index afc14d5..b827c43 100644 --- a/src/ports/postgres/modules/regress/linear.sql_in +++ b/src/ports/postgres/modules/regress/linear.sql_in @@ -646,7 +646,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_transition( RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -656,14 +656,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_merge_states( RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_final( state MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.linregr_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- hetero @@ -675,7 +675,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_transition( RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -685,7 +685,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_merge_states( RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -694,7 +694,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hetero_linregr_final( state MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.heteroskedasticity_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------------------------------------------------- @@ -810,7 +810,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_predict( col_ind_var DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'array_dot' -LANGUAGE c IMMUTABLE STRICT; +LANGUAGE c IMMUTABLE STRICT NOT FENCED; -- Help messages ------------------------------------------------------- CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linregr_predict( diff --git a/src/ports/postgres/modules/regress/logistic.sql_in b/src/ports/postgres/modules/regress/logistic.sql_in index e15b416..5fc9ad1 100644 --- a/src/ports/postgres/modules/regress/logistic.sql_in +++ b/src/ports/postgres/modules/regress/logistic.sql_in @@ -623,7 +623,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_step_transition( DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_cg_step_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -635,7 +635,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_step_transition( DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_irls_step_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -647,7 +647,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_step_transition( DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_igd_step_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -657,7 +657,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_step_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_cg_step_merge_states' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE NOT FENCED STRICT m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -667,7 +667,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_step_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_irls_step_merge_states' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE NOT FENCED STRICT m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -677,7 +677,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_step_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_igd_step_merge_states' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE NOT FENCED STRICT m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -686,7 +686,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_step_final( state DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_cg_step_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE NOT FENCED STRICT m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -695,7 +695,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_step_final( state DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_irls_step_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE NOT FENCED STRICT m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -704,7 +704,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_step_final( state DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'logregr_igd_step_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE NOT FENCED STRICT m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -776,7 +776,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_step_distance( /*+ state2 */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'internal_logregr_cg_step_distance' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -785,7 +785,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_cg_result( /*+ state */ DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.__logregr_result AS 'MODULE_PATHNAME', 'internal_logregr_cg_result' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -795,7 +795,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_step_distance( /*+ state2 */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'internal_logregr_irls_step_distance' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -804,7 +804,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_irls_result( /*+ state */ DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.__logregr_result AS 'MODULE_PATHNAME', 'internal_logregr_irls_result' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -814,7 +814,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_step_distance( /*+ state2 */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'internal_logregr_igd_step_distance' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -823,7 +823,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_igd_result( /*+ state */ DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.__logregr_result AS 'MODULE_PATHNAME', 'internal_logregr_igd_result' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------ @@ -1044,7 +1044,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logregr_predict( col_ind_var DOUBLE PRECISION[] ) RETURNS BOOLEAN AS 'MODULE_PATHNAME', 'logregr_predict' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- Help messages ------------------------------------------------------- @@ -1081,7 +1081,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logregr_predict_prob( col_ind_var DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'logregr_predict_prob' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -- Help messages ------------------------------------------------------- diff --git a/src/ports/postgres/modules/regress/marginal.sql_in b/src/ports/postgres/modules/regress/marginal.sql_in index a19424e..0b459af 100644 --- a/src/ports/postgres/modules/regress/marginal.sql_in +++ b/src/ports/postgres/modules/regress/marginal.sql_in @@ -510,7 +510,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.marginal_logregr_step_transition( coef DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -520,7 +520,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.marginal_logregr_step_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -529,7 +529,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.marginal_logregr_step_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.marginal_logregr_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -785,7 +785,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_marginal_step_transition( coef DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_marginal_step_merge_states( @@ -793,7 +793,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_marginal_step_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -801,7 +801,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_marginal_step_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.marginal_mlogregr_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1207,7 +1207,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__sub_array( index_array INTEGER[] -- The array containing the index ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ----------------------------------------------------------------------- @@ -1233,7 +1233,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_linregr_int_transition( derivative DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'margins_linregr_int_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1243,7 +1243,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_linregr_int_merge( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'margins_linregr_int_merge' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1252,7 +1252,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_linregr_int_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.margins_result AS 'MODULE_PATHNAME', 'margins_linregr_int_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1299,7 +1299,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_logregr_int_transition( x_unset DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'margins_logregr_int_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1309,7 +1309,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_logregr_int_merge( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'margins_logregr_int_merge' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1318,7 +1318,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_logregr_int_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.margins_result AS 'MODULE_PATHNAME', 'margins_logregr_int_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1375,7 +1375,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_mlogregr_int_transition( ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'margins_mlogregr_int_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1385,7 +1385,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_mlogregr_int_merge( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'margins_mlogregr_int_merge' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1394,7 +1394,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_mlogregr_int_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.margins_result AS 'MODULE_PATHNAME', 'margins_mlogregr_int_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1450,7 +1450,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_coxph_int_transition( ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'margins_coxph_int_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1460,7 +1460,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_coxph_int_merge( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'margins_coxph_int_merge' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1469,7 +1469,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_coxph_int_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.margins_result AS 'MODULE_PATHNAME', 'margins_coxph_int_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------------- @@ -1479,7 +1479,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__margins_compute_stats( std_err DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.margins_result AS 'MODULE_PATHNAME', 'margins_compute_stats' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** diff --git a/src/ports/postgres/modules/regress/multilogistic.sql_in b/src/ports/postgres/modules/regress/multilogistic.sql_in index a4acf60..1f4f92b 100644 --- a/src/ports/postgres/modules/regress/multilogistic.sql_in +++ b/src/ports/postgres/modules/regress/multilogistic.sql_in @@ -523,7 +523,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_irls_step_transition ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -534,7 +534,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_irls_step_merge_states ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -544,7 +544,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_irls_step_final ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -575,14 +575,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__internal_mlogregr_irls_step_distance( /*+ state2 */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__internal_mlogregr_irls_result( /*+ state */ DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.mlogregr_result AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -590,7 +590,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__internal_mlogregr_summary_results( /*+ state */ DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.mlogregr_summary_result AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); /** @@ -1071,7 +1071,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_format( ref_category INTEGER ) RETURNS SETOF MADLIB_SCHEMA.__mlogregr_cat_coef AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -1084,7 +1084,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_predict_prob( col_ind_var DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'mlogregr_predict_prob' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -1097,7 +1097,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__mlogregr_predict_response( col_ind_var DOUBLE PRECISION[] ) RETURNS INTEGER AS 'MODULE_PATHNAME', 'mlogregr_predict_response' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------------------- diff --git a/src/ports/postgres/modules/regress/robust.sql_in b/src/ports/postgres/modules/regress/robust.sql_in index 7964eb0..c5ecb7b 100644 --- a/src/ports/postgres/modules/regress/robust.sql_in +++ b/src/ports/postgres/modules/regress/robust.sql_in @@ -531,7 +531,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_linregr_transition( RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- @@ -542,7 +542,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_linregr_merge_states( RETURNS MADLIB_SCHEMA.bytea8 AS 'MODULE_PATHNAME' LANGUAGE C -IMMUTABLE STRICT +IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- @@ -551,7 +551,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_linregr_final( state MADLIB_SCHEMA.bytea8) RETURNS MADLIB_SCHEMA.robust_linregr_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- @@ -646,7 +646,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_logregr_step_transition( coef DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- @@ -656,7 +656,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_logregr_step_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- @@ -665,7 +665,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.robust_logregr_step_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.robust_logregr_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- @@ -864,7 +864,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_robust_step_transition ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- @@ -876,7 +876,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_robust_step_merge_states ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- @@ -887,7 +887,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlogregr_robust_step_final ) RETURNS MADLIB_SCHEMA.robust_mlogregr_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------- diff --git a/src/ports/postgres/modules/sample/sample.sql_in b/src/ports/postgres/modules/sample/sample.sql_in index 8f8a56f..3652f4c 100644 --- a/src/ports/postgres/modules/sample/sample.sql_in +++ b/src/ports/postgres/modules/sample/sample.sql_in @@ -69,7 +69,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_transition_int64( AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -80,7 +80,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_merge_int64( AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -90,7 +90,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_final_int64( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -128,7 +128,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_transition_vector( AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -139,7 +139,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_merge_vector( AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -149,7 +149,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.weighted_sample_final_vector( AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -173,7 +173,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.poisson_random( AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------------------------------------------------- @@ -183,7 +183,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.gamma_random( AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE -STRICT +STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); --------------------------------------------------------------------------- @@ -191,5 +191,5 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.index_weighted_sample( double precision[] ) RETURNS integer AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/sample/stratified_sample.py_in b/src/ports/postgres/modules/sample/stratified_sample.py_in index 90d12d5..7fe88f0 100644 --- a/src/ports/postgres/modules/sample/stratified_sample.py_in +++ b/src/ports/postgres/modules/sample/stratified_sample.py_in @@ -88,9 +88,9 @@ def stratified_sample(schema_madlib, source_table, output_table, proportion, # Find the cut-off label for the given proportion sql2 = """ CREATE TEMP TABLE {perc} AS ( - SELECT {grouping_cols}, percentile_disc({proportion}) - WITHIN GROUP (ORDER BY __samp_out_label) AS __samp_out_label - FROM {label} GROUP BY {grouping_cols})""".format(**locals()) + SELECT {grouping_cols}, + {schema_madlib}.percentile_disc(array_agg(__samp_out_label),{proportion}) AS __samp_out_label + FROM {label} where __samp_out_label is not null GROUP BY {grouping_cols})""".format(**locals()) plpy.execute(sql2) # Select every record that has a label under the threshold diff --git a/src/ports/postgres/modules/stats/clustered_variance_coxph.sql_in b/src/ports/postgres/modules/stats/clustered_variance_coxph.sql_in index 58d6e8e..4475071 100644 --- a/src/ports/postgres/modules/stats/clustered_variance_coxph.sql_in +++ b/src/ports/postgres/modules/stats/clustered_variance_coxph.sql_in @@ -55,7 +55,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_a_b_transition( /* H values */ DOUBLE PRECISION[], /* S */ DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -65,7 +65,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_a_b_merge( /* left state */ DOUBLE PRECISION[], /* right state */ DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -81,7 +81,7 @@ CREATE TYPE MADLIB_SCHEMA.__coxph_a_b_result AS( CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_a_b_final( /* right state */ DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__coxph_a_b_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -117,7 +117,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_compute_w( A DOUBLE PRECISION, B DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ---------------------------------------------------------------------- @@ -136,7 +136,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_compute_clustered_stats( hessian DOUBLE PRECISION[], A DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__coxph_cl_var_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/stats/correlation.sql_in b/src/ports/postgres/modules/stats/correlation.sql_in index 4932385..b9b6f9d 100644 --- a/src/ports/postgres/modules/stats/correlation.sql_in +++ b/src/ports/postgres/modules/stats/correlation.sql_in @@ -396,7 +396,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.correlation_transition( mean double precision[] ) RETURNS double precision[] AS 'MODULE_PATHNAME', 'correlation_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.correlation_merge( @@ -404,14 +404,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.correlation_merge( right_state double precision[] ) RETURNS double precision[] AS 'MODULE_PATHNAME', 'correlation_merge_states' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.correlation_final( state double precision[] ) RETURNS double precision[] AS 'MODULE_PATHNAME', 'correlation_final' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.correlation_agg( diff --git a/src/ports/postgres/modules/stats/cox_prop_hazards.sql_in b/src/ports/postgres/modules/stats/cox_prop_hazards.sql_in index e8faa01..b3b70b6 100644 --- a/src/ports/postgres/modules/stats/cox_prop_hazards.sql_in +++ b/src/ports/postgres/modules/stats/cox_prop_hazards.sql_in @@ -589,7 +589,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_avg_transition( use_abs BOOLEAN ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_avg_merge( @@ -597,14 +597,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_avg_merge( right DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_avg_final( state DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.array_avg( @@ -811,7 +811,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._coxph_predict_resp( pred_type TEXT ) RETURNS DOUBLE PRECISION AS 'MODULE_PATHNAME', 'coxph_predict_resp' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -821,7 +821,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._coxph_predict_terms( mean_ind_var DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'coxph_predict_terms' -LANGUAGE C STRICT IMMUTABLE +LANGUAGE C STRICT IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -858,7 +858,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._split_transition( /* num_splits */ INTEGER ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'split_transition' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------- @@ -868,7 +868,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._split_merge( /* right state */ DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'split_merge' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------- @@ -877,7 +877,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._split_final( /* final state */ DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'split_final' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------- @@ -908,7 +908,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA._compute_grpid( reverse BOOLEAN ) RETURNS INTEGER AS 'MODULE_PATHNAME', 'compute_grpid' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------- @@ -948,7 +948,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.compute_coxph_result( stds DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.coxph_result AS 'MODULE_PATHNAME', 'compute_coxph_result' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -968,7 +968,7 @@ MADLIB_SCHEMA.coxph_improved_step_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.coxph_step_result AS 'MODULE_PATHNAME', 'coxph_improved_step_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -984,7 +984,7 @@ MADLIB_SCHEMA.coxph_improved_step_transition( ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'coxph_improved_step_transition' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------ @@ -1035,7 +1035,7 @@ DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.coxph_improved_strata_step_inner( CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_step_inner_final( state DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ---------------------------------------------------------------------- @@ -1044,7 +1044,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_step_outer_transition( /*+ state1 */ DOUBLE PRECISION[], /*+ state2 */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------------------------------ @@ -1071,7 +1071,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_improved_strata_step_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.coxph_step_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); ------------------------------------ @@ -1149,7 +1149,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__zph_transition( /*+ coef */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'zph_transition' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1159,7 +1159,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__zph_merge( /*+ right_state */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'zph_merge' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1167,7 +1167,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__zph_final( /*+ left_state */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'zph_final' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1199,7 +1199,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__coxph_scale_resid( resid DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'coxph_scale_resid' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1213,7 +1213,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_elem_corr_transition( /*+ y */ DOUBLE PRECISION) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'array_elem_corr_transition' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1223,7 +1223,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_elem_corr_merge( /*+ right_state */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'array_elem_corr_merge' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1231,7 +1231,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_elem_corr_final( /*+ state */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'array_elem_corr_final' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1265,7 +1265,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__coxph_resid_stat_transition( /*+ m */ INTEGER) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'coxph_resid_stat_transition' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1275,7 +1275,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__coxph_resid_stat_merge( /*+ state2 */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'coxph_resid_stat_merge' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1292,7 +1292,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__coxph_resid_stat_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.__cox_resid_stat_result AS 'MODULE_PATHNAME', 'coxph_resid_stat_final' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1479,7 +1479,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_element_min( ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'array_element_min' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -1503,7 +1503,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__array_element_max( ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'array_element_max' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/stats/distribution.sql_in b/src/ports/postgres/modules/stats/distribution.sql_in index 30ee3cd..a60ccb8 100644 --- a/src/ports/postgres/modules/stats/distribution.sql_in +++ b/src/ports/postgres/modules/stats/distribution.sql_in @@ -14,14 +14,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.vectorized_distribution_transition( levels integer[] ) RETURNS double precision[][] AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.vectorized_distribution_final( state double precision[][] ) RETURNS double precision[][] AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.vectorized_distribution_agg( @@ -45,14 +45,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.discrete_distribution_transition( level integer ) RETURNS double precision[] AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE +LANGUAGE c IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.discrete_distribution_final( state double precision[] ) RETURNS double precision[] AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE c IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__',`NO SQL', `'); DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.discrete_distribution_agg( diff --git a/src/ports/postgres/modules/stats/hypothesis_tests.sql_in b/src/ports/postgres/modules/stats/hypothesis_tests.sql_in index 61dbb89..ba43a03 100644 --- a/src/ports/postgres/modules/stats/hypothesis_tests.sql_in +++ b/src/ports/postgres/modules/stats/hypothesis_tests.sql_in @@ -572,7 +572,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_one_transition( value DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_merge_states( @@ -580,7 +580,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); @@ -588,7 +588,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_one_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.t_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); DROP TYPE IF EXISTS MADLIB_SCHEMA.f_test_result CASCADE; @@ -604,7 +604,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.f_test_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.f_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); /** @@ -664,7 +664,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_two_transition( "value" DOUBLE PRECISION) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); @@ -672,7 +672,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_two_pooled_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.t_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); /** * @brief Perform two-sample pooled (i.e., equal variances) Student t-test @@ -739,7 +739,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.t_test_two_unpooled_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.t_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); /** * @brief Perform unpooled (i.e., unequal variances) t-test (also known as @@ -861,7 +861,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition( df BIGINT ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition( @@ -870,7 +870,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition( expected DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition( @@ -878,7 +878,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_transition( observed BIGINT ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_merge_states( @@ -886,7 +886,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); DROP TYPE IF EXISTS MADLIB_SCHEMA.chi2_test_result CASCADE; @@ -902,7 +902,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.chi2_gof_test_final( state DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.chi2_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); /** @@ -1009,7 +1009,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.ks_test_transition( "numSecond" BIGINT ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); DROP TYPE IF EXISTS MADLIB_SCHEMA.ks_test_result CASCADE; @@ -1023,7 +1023,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.ks_test_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.ks_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); /** @@ -1097,7 +1097,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mw_test_transition( "value" DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); DROP TYPE IF EXISTS MADLIB_SCHEMA.mw_test_result CASCADE; @@ -1112,7 +1112,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mw_test_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.mw_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); /** * @brief Perform Mann-Whitney test @@ -1184,7 +1184,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.wsr_test_transition( "precision" DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.wsr_test_transition( @@ -1192,7 +1192,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.wsr_test_transition( value DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); @@ -1211,7 +1211,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.wsr_test_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.wsr_test_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); /** * @brief Perform Wilcoxon-Signed-Rank test @@ -1336,7 +1336,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.one_way_anova_transition( value DOUBLE PRECISION) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.one_way_anova_merge_states( @@ -1344,14 +1344,14 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.one_way_anova_merge_states( state2 DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.one_way_anova_final( state DOUBLE PRECISION[]) RETURNS MADLIB_SCHEMA.one_way_anova_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(, , ); /** * @brief Perform one-way analysis of variance diff --git a/src/ports/postgres/modules/stats/robust_variance_coxph.sql_in b/src/ports/postgres/modules/stats/robust_variance_coxph.sql_in index 5bc58f7..710a649 100644 --- a/src/ports/postgres/modules/stats/robust_variance_coxph.sql_in +++ b/src/ports/postgres/modules/stats/robust_variance_coxph.sql_in @@ -60,7 +60,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_coxph_step_transition( /* H values */ DOUBLE PRECISION[], /* S */ DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -69,7 +69,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_coxph_step_final( /* state */ DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__rb_coxph_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------- @@ -106,7 +106,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_h_s_transition( /* x */ DOUBLE PRECISION[], /* coef */ DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------- @@ -115,7 +115,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_h_s_merge( /*+ left_state */ DOUBLE PRECISION[], /*+ right_state */ DOUBLE PRECISION[]) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------- @@ -123,7 +123,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.coxph_h_s_final( /* state */ DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__rb_coxph_hs_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------- @@ -148,7 +148,7 @@ CREATE AGGREGATE MADLIB_SCHEMA.coxph_h_s( CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_coxph_strata_step_final( /* state */ DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------- @@ -185,7 +185,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_sum_strata_transition( in_state1 DOUBLE PRECISION[], in_state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------- @@ -193,7 +193,7 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.rb_sum_strata_final( in_state DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__rb_coxph_result AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); -------------------------------- diff --git a/src/ports/postgres/modules/summary/Summarizer.py_in b/src/ports/postgres/modules/summary/Summarizer.py_in index 5790972..a4030d3 100644 --- a/src/ports/postgres/modules/summary/Summarizer.py_in +++ b/src/ports/postgres/modules/summary/Summarizer.py_in @@ -169,17 +169,19 @@ class Summarizer: return "NULL" def mfv_type(get_count, c): - slicing = ('0:0', '1:1')[get_count] + slicing = ('0,0', '1,1')[get_count] mfv_method = ('mfvsketch_top_histogram', 'mfvsketch_quick_histogram')[self._get_mfv_quick] + dim_end = self._how_many_mfv - 1 return """ array_to_string(({schema_madlib}.{mfv_method}( - {column},{topk}))[0:{topk}-1][{slice}], + {column},{topk}))[0,{dim_end}][{slice}], '{delimiter}')""".format( schema_madlib=self._schema_madlib, mfv_method=mfv_method, column=c['attname'], topk=self._how_many_mfv, + dim_end=dim_end, slice=slicing, delimiter=self._delimiter) # ------ End of Helper sub-functions ------ diff --git a/src/ports/postgres/modules/svm/svm.sql_in b/src/ports/postgres/modules/svm/svm.sql_in index ba05e86..8a538aa 100644 --- a/src/ports/postgres/modules/svm/svm.sql_in +++ b/src/ports/postgres/modules/svm/svm.sql_in @@ -1155,20 +1155,20 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linear_svm_igd_transition( tuple_weight double precision ) RETURNS double precision[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linear_svm_igd_merge( state1 double precision[], state2 double precision[]) RETURNS double precision[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.linear_svm_igd_final( state double precision[]) RETURNS double precision[] AS 'MODULE_PATHNAME' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); /** @@ -1213,13 +1213,13 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_linear_svm_igd_distance( /*+ state1 */ double precision[], /*+ state2 */ double precision[]) RETURNS double precision AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.internal_linear_svm_igd_result( /*+ state */ double precision[]) RETURNS MADLIB_SCHEMA.linear_svm_result AS 'MODULE_PATHNAME' -LANGUAGE c IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL'); diff --git a/src/ports/postgres/modules/tsa/arima.py_in b/src/ports/postgres/modules/tsa/arima.py_in index 6ae7588..2dfea47 100644 --- a/src/ports/postgres/modules/tsa/arima.py_in +++ b/src/ports/postgres/modules/tsa/arima.py_in @@ -202,7 +202,7 @@ def arima_train (schema_madlib, input_table, output_table, # Also add an auxiliary array column which contains p previous time # series values. # tbl_ts is the time series table we are going to use. For convenience, - # we fix the column names: tid, distid and tvals for time, distribution + # we fix the column names: timeid, distid and tvals for time, distribution # id and an array of current value and p previous values. # if the data has been differenced, we only need to do adjustment if d > 0: @@ -621,16 +621,16 @@ def _redist_data(schema_madlib, input_table, timestamp_col, dist_table = unique_string() + "redist__" # Redistribute the data - # + # plpy.execute( """ create temp table {dist_table} as select - ((tid - 1) / {chunk_size})::integer + 1 as distid, - tid, origin_tid, tval + ((timeid - 1) / {chunk_size})::integer + 1 as distid, + timeid, origin_tid, tval from ( select - row_number() over (order by {timestamp_col}) as tid, + row_number() over (order by {timestamp_col}) as timeid, {timestamp_col} as origin_tid, {timeseries_col} as tval from @@ -650,10 +650,10 @@ def _redist_data(schema_madlib, input_table, timestamp_col, """ insert into {dist_table} select - o.distid + 1, tid, NULL, tval + o.distid + 1, timeid, NULL, tval from ( select - distid, max(tid) maxid + distid, max(timeid) maxid from {dist_table} where @@ -662,24 +662,24 @@ def _redist_data(schema_madlib, input_table, timestamp_col, ) q1, {dist_table} o where q1.distid = o.distid and - q1.maxid - o.tid < {d} + q1.maxid - o.timeid < {d} """.format(dist_table=dist_table, d=d)) output_table = unique_string() + "redist_agg__" # Aggregate the chunk into an array # Note that each aggregated chunk except for the first one - # has additonal p or d elements and tid is 1 for the first - # chunk and is the actual starting tid for the other chunks + # has additonal p or d elements and timeid is 1 for the first + # chunk and is the actual starting timeid for the other chunks # - the corresponding tval is the pth (0 based) element in # tvals - # Note that both distid and tid starts from 1 + # Note that both distid and timeid starts from 1 plpy.execute( """ create temp table {output_table} as select distid, - (distid - 1) * {chunk_size} + 1 as tid, - array_agg(tval order by tid) as tvals + (distid - 1) * {chunk_size} + 1 as timeid, + array_agg(tval order by timeid) as tvals from {dist_table} group by @@ -722,14 +722,14 @@ def _diff (schema_madlib, input_table, timestamp_col, timeseries_col, d, create temp table {output_table} as select distid::integer, - case when tid = 1 then 1::integer - else (tid - {d})::integer - end as tid, + case when timeid = 1 then 1::integer + else (timeid - {d})::integer + end as timeid, tvals from ( select - distid, tid, + distid, timeid, {schema_madlib}.__arima_diff(tvals, {d}) tvals from {input_table} @@ -759,7 +759,7 @@ def _adjust_diff_data(schema_madlib, diff_table, p): """ create temp table {output_table} as select - t1.distid::integer, t1.tid::integer, + t1.distid::integer, t1.timeid::integer, {schema_madlib}.__arima_adjust(t1.distid, t1.tvals, t2.tvals, {p}) as tvals from {diff_table} as t1, {diff_table} as t2 @@ -816,7 +816,7 @@ def _create_resid_tbl (schema_madlib, tbl_ts, p, d, q, phi, if q != 0: theta_str = "array[" + ",".join(str(i) for i in theta) + "]" - # Joining on tid is really slow on large data + # Joining on timeid is really slow on large data # (more than 100 times slower). # This is why we use join on distid, which gives almost the # same performance as that of the original code. @@ -1099,7 +1099,7 @@ def _compute_statistics (schema_madlib, tbl_ts, p, q, phi, theta, Args: @param schema_madlib - MADlib schema name - @param tbl_ts - time series table name, it has two columns: tid + @param tbl_ts - time series table name, it has two columns: timeid and tvals, which is the p previous values plus the current value ordered by time. @param p - integer, AR order @@ -1129,7 +1129,7 @@ def _compute_statistics (schema_madlib, tbl_ts, p, q, phi, theta, '{theta}'::double precision[], {mean}, {delta} - order by tid + order by timeid ) as f from {tbl_ts} ) s diff --git a/src/ports/postgres/modules/tsa/arima.sql_in b/src/ports/postgres/modules/tsa/arima.sql_in index 48f0abd..5769ca3 100644 --- a/src/ports/postgres/modules/tsa/arima.sql_in +++ b/src/ports/postgres/modules/tsa/arima.sql_in @@ -641,7 +641,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_residual prez DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_residual' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -652,7 +652,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_diff d INTEGER ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_diff' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -666,7 +666,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_adjust p INTEGER ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_adjust' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -680,7 +680,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_delta u DOUBLE PRECISION -- Weight of gradient-descent step ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_lm_delta' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -713,7 +713,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm ) RETURNS MADLIB_SCHEMA.__arima_lm_result AS 'MODULE_PATHNAME', 'arima_lm' -LANGUAGE C +LANGUAGE C NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -728,7 +728,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_result_sfunc ( z2 DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_lm_result_sfunc' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -739,7 +739,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_result_pfunc ( state2 DOUBLE PRECISION[] ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_lm_result_pfunc' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -759,7 +759,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_result_ffunc ( state_data DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__arima_lm_sum_result AS 'MODULE_PATHNAME', 'arima_lm_result_ffunc' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -799,7 +799,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_stat_sfunc ( delta DOUBLE PRECISION ) RETURNS DOUBLE PRECISION[] AS 'MODULE_PATHNAME', 'arima_lm_stat_sfunc' -LANGUAGE C IMMUTABLE +LANGUAGE C IMMUTABLE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); @@ -818,7 +818,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__arima_lm_stat_ffunc ( state_data DOUBLE PRECISION[] ) RETURNS MADLIB_SCHEMA.__arima_lm_stat_result AS 'MODULE_PATHNAME', 'arima_lm_stat_ffunc' -LANGUAGE C IMMUTABLE STRICT +LANGUAGE C IMMUTABLE STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/utilities/control.py_in b/src/ports/postgres/modules/utilities/control.py_in index 32f2f59..3965e50 100644 --- a/src/ports/postgres/modules/utilities/control.py_in +++ b/src/ports/postgres/modules/utilities/control.py_in @@ -234,7 +234,7 @@ class AOControl(ContextDecorator): def __init__(self, enable=False): self.to_enable = enable self.was_ao_enabled = False - self.guc_exists = True + self.guc_exists = False self.storage_options_dict = dict() def _parse_gp_default_storage_options(self, gp_default_storage_options_str): diff --git a/src/ports/postgres/modules/utilities/in_mem_group_control.py_in b/src/ports/postgres/modules/utilities/in_mem_group_control.py_in index 86f0c17..be28ea0 100644 --- a/src/ports/postgres/modules/utilities/in_mem_group_control.py_in +++ b/src/ports/postgres/modules/utilities/in_mem_group_control.py_in @@ -106,10 +106,14 @@ class BaseState(object): if col_grp_state == '': self._is_none = True for s in ret_states: + if s[col_grp_key] is None: + s[col_grp_key] = ' ' self._state[s[col_grp_key]] = None return failed_grp_keys for t in ret_states: _grp_key, _grp_state = t[col_grp_key], t[col_grp_state] + if _grp_key is None: + _grp_key = ' ' if _grp_state is None: failed_grp_keys.append(_grp_key) else: @@ -204,6 +208,7 @@ class GroupIterationController: col_n_tuples: Name of the column containing count of tuples. Set to a unique string if not defined """ + NULL_list = ['Null', 'NULL','None','NONE'] self.schema_madlib = arg_dict['schema_madlib'] self.in_with = False self.iteration = -1 @@ -222,6 +227,8 @@ class GroupIterationController: if arg_dict["grouping_col"] is None else arg_dict["grouping_col"]), ) + if self.kwargs['grouping_str'] in NULL_list: + self.kwargs['grouping_str'] = "' '" self.grp_to_n_tuples = {} self.failed_grp_keys = [] @@ -287,6 +294,7 @@ class GroupIterationController: else "{grouping_col}").format(**self.kwargs) groupby_str = ("{col_grp_null}" if self.is_group_null else "{grouping_col}").format(**self.kwargs) + plpy.execute(""" DROP TABLE IF EXISTS {rel_state}; CREATE TEMPORARY TABLE {rel_state} AS ( @@ -384,6 +392,7 @@ class GroupIterationController: select_rel_state=group_param.select_rel_state, select_n_tuples=group_param.select_n_tuples, **self.kwargs) + insert_plan = plpy.prepare(insert_sql, ["text[]", group_param.grouped_state_type, "text[]", "bigint[]"]) diff --git a/src/ports/postgres/modules/utilities/minibatch_preprocessing.py_in b/src/ports/postgres/modules/utilities/minibatch_preprocessing.py_in index c25463c..9346d4a 100644 --- a/src/ports/postgres/modules/utilities/minibatch_preprocessing.py_in +++ b/src/ports/postgres/modules/utilities/minibatch_preprocessing.py_in @@ -284,8 +284,8 @@ class MiniBatchPreProcessor: {self.schema_madlib}.matrix_agg({dep_colname}) as {dep_colname}, {self.schema_madlib}.matrix_agg({ind_colname}) as {ind_colname} FROM ( - SELECT (row_number() OVER ({partition_by} ORDER BY random()) - 1) - / {buffer_size} + SELECT trunc((row_number() OVER ({partition_by} ORDER BY random()) - 1) + / {buffer_size}) as {row_id}, * FROM ( {standardize_query} @@ -474,7 +474,7 @@ class MiniBatchBufferSizeCalculator: default_buffer_size to int, because it will be used to calculate the row id of the packed input. The query looks like this - SELECT (row_number() OVER (ORDER BY random()) - 1) / {buffer_size} + SELECT (row_number() OVER (ORDER BY random())) / {buffer_size} This calculation has to return an int for which buffer_size has to be an int diff --git a/src/ports/postgres/modules/utilities/path.sql_in b/src/ports/postgres/modules/utilities/path.sql_in index 6c102eb..6ab554a 100644 --- a/src/ports/postgres/modules/utilities/path.sql_in +++ b/src/ports/postgres/modules/utilities/path.sql_in @@ -551,7 +551,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.path_pattern_match overlapping_patterns BOOLEAN ) RETURNS MADLIB_SCHEMA.path_match_result AS 'MODULE_PATHNAME', 'path_pattern_match' -LANGUAGE C STRICT +LANGUAGE C STRICT NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `'); diff --git a/src/ports/postgres/modules/utilities/pivot.py_in b/src/ports/postgres/modules/utilities/pivot.py_in index 1b1a5d9..48a8376 100644 --- a/src/ports/postgres/modules/utilities/pivot.py_in +++ b/src/ports/postgres/modules/utilities/pivot.py_in @@ -305,8 +305,7 @@ def pivot(schema_madlib, source_table, output_table, index, pivot_cols, # Aggregate over each pivot column, while filtering all NULL # values created by previous query. sub_pivot_str_sel = _fill_value_wrapper( - "{agg}({p_name}) " - " FILTER (WHERE {p_name} IS NOT NULL)". + "{agg}(CASE WHEN {p_name} IS NOT NULL THEN {p_name} END )". format(agg=agg, p_name=p_name)) if not is_array_output: # keep spaces around the 'AS' diff --git a/src/ports/postgres/modules/utilities/test/minibatch_preprocessing.sql_in b/src/ports/postgres/modules/utilities/test/minibatch_preprocessing.sql_in index b6b2996..3f7d0f1 100644 --- a/src/ports/postgres/modules/utilities/test/minibatch_preprocessing.sql_in +++ b/src/ports/postgres/modules/utilities/test/minibatch_preprocessing.sql_in @@ -139,7 +139,7 @@ SELECT assert ) from (select * from minibatch_preprocessing_out_summary) summary; -- grouping with the same dataset -\set expected_grouping_row_count '\'' 1,1,2 '\'' +\set expected_grouping_row_count '''1,1,2''' DROP TABLE IF EXISTS minibatch_preprocessing_out, minibatch_preprocessing_out_standardization, minibatch_preprocessing_out_summary; SELECT minibatch_preprocessor('minibatch_preprocessing_input', 'minibatch_preprocessing_out', 'length>0.2', 'ARRAY[diameter,height,whole,shucked,viscera,shell]', 'rings', 4); SELECT assert @@ -231,8 +231,8 @@ INSERT INTO minibatch_preprocessing_input(x1,x2,y) VALUES SELECT minibatch_preprocessor('minibatch_preprocessing_input', 'minibatch_preprocessing_out', 'y', 'ARRAY[x1,x2]', NULL, 2); -- since the order is not deterministic, we assert for all possible orders -\set expected_normalized_independent_var1 '\'' {{-1, -1},{1, 1}} '\'' -\set expected_normalized_independent_var2 '\'' {{1, 1},{-1, -1}} '\'' +\set expected_normalized_independent_var1 '''{{-1,-1},{1,1}}''' +\set expected_normalized_independent_var2 '''{{1,1},{-1,-1}}''' SELECT assert ( diff --git a/src/ports/postgres/modules/utilities/test/pivot.sql_in b/src/ports/postgres/modules/utilities/test/pivot.sql_in index fb034e7..695805a 100644 --- a/src/ports/postgres/modules/utilities/test/pivot.sql_in +++ b/src/ports/postgres/modules/utilities/test/pivot.sql_in @@ -48,8 +48,8 @@ SELECT assert(val_avg_piv_20 = 3, 'Wrong output in pivoting') FROM pivout WHERE DROP VIEW IF EXISTS pivset_ext; CREATE VIEW pivset_ext AS SELECT *, - COALESCE(id + ("PIVSET".val / 3), 0) AS id2, - COALESCE(piv + ("PIVSET".val / 3), 0) AS piv2, + COALESCE(id + floor("PIVSET".val / 3), 0)::int AS id2, + COALESCE(piv + floor("PIVSET".val / 3), 0)::int AS piv2, COALESCE(val + 10, 0) AS val2 FROM "PIVSET"; SELECT id,id2,piv,piv2,val,val2 FROM pivset_ext diff --git a/src/ports/postgres/modules/utilities/test/sessionize.sql_in b/src/ports/postgres/modules/utilities/test/sessionize.sql_in index 808970c..1d948f3 100644 --- a/src/ports/postgres/modules/utilities/test/sessionize.sql_in +++ b/src/ports/postgres/modules/utilities/test/sessionize.sql_in @@ -75,7 +75,7 @@ INSERT INTO eventlog_installchk VALUES (to_timestamp(NULL, 'MM/DD/YYYY HH:MI:SS'), 103711, NULL, 'WINE', 0, 1, 't'), (to_timestamp('04/15/2016 02:17:00', 'MM/DD/YYYY HH:MI:SS'), 103711, 1, 'BEER', 0, 1, 't'), (to_timestamp('04/15/2016 02:21:00', 'MM/DD/YYYY HH:MI:SS'), 103711, 2, 'LANDING', 0, 1, 't'), -(to_timestamp('04/15/2016 02:31:0.05', 'MM/DD/YYYY HH:MI:SS'), 103711, 3, 'WINE', 0, 1, 't'); +(to_timestamp('04/15/2016 02:31:0.05', 'MM/DD/YYYY HH:MI:SS.FF'), 103711, 3, 'WINE', 0, 1, 't'); SELECT sessionize( diff --git a/src/ports/postgres/modules/utilities/test/transform_vec_cols.sql_in b/src/ports/postgres/modules/utilities/test/transform_vec_cols.sql_in index b43b39f..860d20d 100644 --- a/src/ports/postgres/modules/utilities/test/transform_vec_cols.sql_in +++ b/src/ports/postgres/modules/utilities/test/transform_vec_cols.sql_in @@ -397,7 +397,7 @@ SELECT vec2cols( 'out_table2', 'feature_vector' ); -SELECT assert((select data_type from information_schema.columns where table_name = 'out_table2' limit 1) = 'double precision', 'Incorrect type casting of features for cols2vec'); +SELECT assert((select data_type from information_schema.columns where table_name = 'out_table2' limit 1) = 'numeric', 'Incorrect type casting of features for cols2vec'); DROP TABLE IF EXISTS out_table, out_table_summary, out_table2; SELECT cols2vec( diff --git a/src/ports/postgres/modules/utilities/utilities.py_in b/src/ports/postgres/modules/utilities/utilities.py_in index 577235e..6941926 100644 --- a/src/ports/postgres/modules/utilities/utilities.py_in +++ b/src/ports/postgres/modules/utilities/utilities.py_in @@ -611,6 +611,8 @@ def set_client_min_messages(new_level): def is_pg_major_version_less_than(schema_madlib, compare_version, **kwargs): version = plpy.execute("select version()")[0]["version"] + if "openGauss" in version: + return True regex = re.compile('PostgreSQL\s*([0-9]+)([0-9.beta]+)', re.IGNORECASE) version = regex.findall(version) if len(version) > 0 and int(version[0][0]) < compare_version: diff --git a/src/ports/postgres/modules/utilities/utilities.sql_in b/src/ports/postgres/modules/utilities/utilities.sql_in index bbf861d..2d938e6 100644 --- a/src/ports/postgres/modules/utilities/utilities.sql_in +++ b/src/ports/postgres/modules/utilities/utilities.sql_in @@ -265,11 +265,11 @@ m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `READS SQL DATA', `'); * >PostgreSQL documentation. For portability, MADlib code should not make * use of such "features" directly, but only use isnan() instead. * - * @param number - * @returns \c TRUE if \c number is \c NaN, \c FALSE otherwise + * @param number1 + * @returns \c TRUE if \c number1 is \c NaN, \c FALSE otherwise */ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.isnan( - number DOUBLE PRECISION + number1 DOUBLE PRECISION ) RETURNS BOOLEAN AS $$ SELECT $1 = 'NaN'::DOUBLE PRECISION; @@ -313,7 +313,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.noop() RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE c -VOLATILE +VOLATILE NOT FENCED m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); ------------------------------------------------------------------------ @@ -434,7 +434,7 @@ $$m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `'); CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__to_char(val anyelement) RETURNS TEXT AS 'MODULE_PATHNAME', '__to_text' -LANGUAGE C STRICT IMMUTABLE; +LANGUAGE C STRICT IMMUTABLE NOT FENCED; ------------------------------------------------------------------------ diff --git a/src/ports/postgres/modules/utilities/validate_args.py_in b/src/ports/postgres/modules/utilities/validate_args.py_in index a21be58..6f2f3bd 100644 --- a/src/ports/postgres/modules/utilities/validate_args.py_in +++ b/src/ports/postgres/modules/utilities/validate_args.py_in @@ -215,10 +215,11 @@ def get_first_schema(table_name): def drop_tables(table_list): """ Drop tables specified in table_list. + Notice: Both TEMP and non-TEMP objects maybe in the table_list at the same time. """ - drop_str = ', '.join(table_list) - if drop_str: - plpy.execute("DROP TABLE IF EXISTS {0}".format(drop_str)) + if len(table_list) > 0: + for table in table_list: + plpy.execute("DROP TABLE IF EXISTS {0}".format(table)) def table_is_empty(tbl, filter_str=None): diff --git a/src/ports/postgres/modules/xgboost_gs/__init__.py_in b/src/ports/postgres/modules/xgboost_gs/__init__.py_in new file mode 100644 index 0000000..e69de29 diff --git a/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_classifi.sql_in b/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_classifi.sql_in new file mode 100644 index 0000000..4446481 --- /dev/null +++ b/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_classifi.sql_in @@ -0,0 +1,257 @@ +/* + * Test case for sklearn-api xgboost. + */ + +m4_include(`SQLCommon.m4') + +select MADLIB_SCHEMA.xgboost_sk_Classifier(); +select MADLIB_SCHEMA.xgboost_sk_Classifier('usage'); + +DROP TABLE IF EXISTS iris; +create table iris (id serial, a float, d float, c float, b float, label int); + +INSERT into iris (a, b, c, d, label) values (5.1, 3.5, 1.4, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.9, 3.0, 1.4, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (4.7, 3.2, 1.3, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.6, 3.1, 1.5, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (5.0, 3.6, 1.4, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.4, 3.9, 1.7, 0.4, 0); +INSERT into iris (a, b, c, d, label) values (4.6, 3.4, 1.4, 0.3, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.4, 1.5, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (4.4, 2.9, 1.4, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.9, 3.1, 1.5, 0.1, 0); +INSERT into iris (a, b, c, d, label) values (5.4, 3.7, 1.5, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.8, 3.4, 1.6, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (4.8, 3.0, 1.4, 0.1, 0);INSERT into iris (a, b, c, d, label) values (4.3, 3.0, 1.1, 0.1, 0); +INSERT into iris (a, b, c, d, label) values (5.8, 4.0, 1.2, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.7, 4.4, 1.5, 0.4, 0); +INSERT into iris (a, b, c, d, label) values (5.4, 3.9, 1.3, 0.4, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.5, 1.4, 0.3, 0); +INSERT into iris (a, b, c, d, label) values (5.7, 3.8, 1.7, 0.3, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.8, 1.5, 0.3, 0); +INSERT into iris (a, b, c, d, label) values (5.4, 3.4, 1.7, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.7, 1.5, 0.4, 0); +INSERT into iris (a, b, c, d, label) values (4.6, 3.6, 1.0, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.3, 1.7, 0.5, 0); +INSERT into iris (a, b, c, d, label) values (4.8, 3.4, 1.9, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.0, 1.6, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (5.0, 3.4, 1.6, 0.4, 0);INSERT into iris (a, b, c, d, label) values (5.2, 3.5, 1.5, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (5.2, 3.4, 1.4, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.7, 3.2, 1.6, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (4.8, 3.1, 1.6, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.4, 3.4, 1.5, 0.4, 0); +INSERT into iris (a, b, c, d, label) values (5.2, 4.1, 1.5, 0.1, 0);INSERT into iris (a, b, c, d, label) values (5.5, 4.2, 1.4, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (4.9, 3.1, 1.5, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.2, 1.2, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (5.5, 3.5, 1.3, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.9, 3.6, 1.4, 0.1, 0); +INSERT into iris (a, b, c, d, label) values (4.4, 3.0, 1.3, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.1, 3.4, 1.5, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (5.0, 3.5, 1.3, 0.3, 0);INSERT into iris (a, b, c, d, label) values (4.5, 2.3, 1.3, 0.3, 0); +INSERT into iris (a, b, c, d, label) values (4.4, 3.2, 1.3, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.5, 1.6, 0.6, 0); +INSERT into iris (a, b, c, d, label) values (5.1, 3.8, 1.9, 0.4, 0);INSERT into iris (a, b, c, d, label) values (4.8, 3.0, 1.4, 0.3, 0); +INSERT into iris (a, b, c, d, label) values (5.1, 3.8, 1.6, 0.2, 0);INSERT into iris (a, b, c, d, label) values (4.6, 3.2, 1.4, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (5.3, 3.7, 1.5, 0.2, 0);INSERT into iris (a, b, c, d, label) values (5.0, 3.3, 1.4, 0.2, 0); +INSERT into iris (a, b, c, d, label) values (7.0, 3.2, 4.7, 1.4, 1);INSERT into iris (a, b, c, d, label) values (6.4, 3.2, 4.5, 1.5, 1); +INSERT into iris (a, b, c, d, label) values (6.9, 3.1, 4.9, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.5, 2.3, 4.0, 1.3, 1); +INSERT into iris (a, b, c, d, label) values (6.5, 2.8, 4.6, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.7, 2.8, 4.5, 1.3, 1); +INSERT into iris (a, b, c, d, label) values (6.3, 3.3, 4.7, 1.6, 1);INSERT into iris (a, b, c, d, label) values (4.9, 2.4, 3.3, 1.0, 1); +INSERT into iris (a, b, c, d, label) values (6.6, 2.9, 4.6, 1.3, 1);INSERT into iris (a, b, c, d, label) values (5.2, 2.7, 3.9, 1.4, 1); +INSERT into iris (a, b, c, d, label) values (5.0, 2.0, 3.5, 1.0, 1);INSERT into iris (a, b, c, d, label) values (5.9, 3.0, 4.2, 1.5, 1); +INSERT into iris (a, b, c, d, label) values (6.0, 2.2, 4.0, 1.0, 1);INSERT into iris (a, b, c, d, label) values (6.1, 2.9, 4.7, 1.4, 1); +INSERT into iris (a, b, c, d, label) values (5.6, 2.9, 3.6, 1.3, 1);INSERT into iris (a, b, c, d, label) values (6.7, 3.1, 4.4, 1.4, 1); +INSERT into iris (a, b, c, d, label) values (5.6, 3.0, 4.5, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.8, 2.7, 4.1, 1.0, 1); +INSERT into iris (a, b, c, d, label) values (6.2, 2.2, 4.5, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.6, 2.5, 3.9, 1.1, 1); +INSERT into iris (a, b, c, d, label) values (5.9, 3.2, 4.8, 1.8, 1);INSERT into iris (a, b, c, d, label) values (6.1, 2.8, 4.0, 1.3, 1); +INSERT into iris (a, b, c, d, label) values (6.3, 2.5, 4.9, 1.5, 1);INSERT into iris (a, b, c, d, label) values (6.1, 2.8, 4.7, 1.2, 1); +INSERT into iris (a, b, c, d, label) values (6.4, 2.9, 4.3, 1.3, 1);INSERT into iris (a, b, c, d, label) values (6.6, 3.0, 4.4, 1.4, 1); +INSERT into iris (a, b, c, d, label) values (6.8, 2.8, 4.8, 1.4, 1);INSERT into iris (a, b, c, d, label) values (6.7, 3.0, 5.0, 1.7, 1); +INSERT into iris (a, b, c, d, label) values (6.0, 2.9, 4.5, 1.5, 1);INSERT into iris (a, b, c, d, label) values (5.7, 2.6, 3.5, 1.0, 1); +INSERT into iris (a, b, c, d, label) values (5.5, 2.4, 3.8, 1.1, 1);INSERT into iris (a, b, c, d, label) values (5.5, 2.4, 3.7, 1.0, 1); +INSERT into iris (a, b, c, d, label) values (5.8, 2.7, 3.9, 1.2, 1);INSERT into iris (a, b, c, d, label) values (6.0, 2.7, 5.1, 1.6, 1); +INSERT into iris (a, b, c, d, label) values (5.4, 3.0, 4.5, 1.5, 1);INSERT into iris (a, b, c, d, label) values (6.0, 3.4, 4.5, 1.6, 1); +INSERT into iris (a, b, c, d, label) values (6.7, 3.1, 4.7, 1.5, 1);INSERT into iris (a, b, c, d, label) values (6.3, 2.3, 4.4, 1.3, 1); +INSERT into iris (a, b, c, d, label) values (5.6, 3.0, 4.1, 1.3, 1);INSERT into iris (a, b, c, d, label) values (5.5, 2.5, 4.0, 1.3, 1); +INSERT into iris (a, b, c, d, label) values (5.5, 2.6, 4.4, 1.2, 1);INSERT into iris (a, b, c, d, label) values (6.1, 3.0, 4.6, 1.4, 1); +INSERT into iris (a, b, c, d, label) values (5.8, 2.6, 4.0, 1.2, 1);INSERT into iris (a, b, c, d, label) values (5.0, 2.3, 3.3, 1.0, 1); +INSERT into iris (a, b, c, d, label) values (5.6, 2.7, 4.2, 1.3, 1);INSERT into iris (a, b, c, d, label) values (5.7, 3.0, 4.2, 1.2, 1); +INSERT into iris (a, b, c, d, label) values (5.7, 2.9, 4.2, 1.3, 1);INSERT into iris (a, b, c, d, label) values (6.2, 2.9, 4.3, 1.3, 1); +INSERT into iris (a, b, c, d, label) values (5.1, 2.5, 3.0, 1.1, 1);INSERT into iris (a, b, c, d, label) values (5.7, 2.8, 4.1, 1.3, 1); +INSERT into iris (a, b, c, d, label) values (6.3, 3.3, 6.0, 2.5, 2);INSERT into iris (a, b, c, d, label) values (5.8, 2.7, 5.1, 1.9, 2); +INSERT into iris (a, b, c, d, label) values (7.1, 3.0, 5.9, 2.1, 2);INSERT into iris (a, b, c, d, label) values (6.3, 2.9, 5.6, 1.8, 2); +INSERT into iris (a, b, c, d, label) values (6.5, 3.0, 5.8, 2.2, 2);INSERT into iris (a, b, c, d, label) values (7.6, 3.0, 6.6, 2.1, 2); +INSERT into iris (a, b, c, d, label) values (4.9, 2.5, 4.5, 1.7, 2);INSERT into iris (a, b, c, d, label) values (7.3, 2.9, 6.3, 1.8, 2); +INSERT into iris (a, b, c, d, label) values (6.7, 2.5, 5.8, 1.8, 2);INSERT into iris (a, b, c, d, label) values (7.2, 3.6, 6.1, 2.5, 2); +INSERT into iris (a, b, c, d, label) values (6.5, 3.2, 5.1, 2.0, 2);INSERT into iris (a, b, c, d, label) values (6.4, 2.7, 5.3, 1.9, 2); +INSERT into iris (a, b, c, d, label) values (6.8, 3.0, 5.5, 2.1, 2);INSERT into iris (a, b, c, d, label) values (5.7, 2.5, 5.0, 2.0, 2); +INSERT into iris (a, b, c, d, label) values (5.8, 2.8, 5.1, 2.4, 2);INSERT into iris (a, b, c, d, label) values (6.4, 3.2, 5.3, 2.3, 2); +INSERT into iris (a, b, c, d, label) values (6.5, 3.0, 5.5, 1.8, 2);INSERT into iris (a, b, c, d, label) values (7.7, 3.8, 6.7, 2.2, 2); +INSERT into iris (a, b, c, d, label) values (7.7, 2.6, 6.9, 2.3, 2);INSERT into iris (a, b, c, d, label) values (6.0, 2.2, 5.0, 1.5, 2); +INSERT into iris (a, b, c, d, label) values (6.9, 3.2, 5.7, 2.3, 2);INSERT into iris (a, b, c, d, label) values (5.6, 2.8, 4.9, 2.0, 2); +INSERT into iris (a, b, c, d, label) values (7.7, 2.8, 6.7, 2.0, 2);INSERT into iris (a, b, c, d, label) values (6.3, 2.7, 4.9, 1.8, 2); +INSERT into iris (a, b, c, d, label) values (6.7, 3.3, 5.7, 2.1, 2);INSERT into iris (a, b, c, d, label) values (7.2, 3.2, 6.0, 1.8, 2); +INSERT into iris (a, b, c, d, label) values (6.2, 2.8, 4.8, 1.8, 2);INSERT into iris (a, b, c, d, label) values (6.1, 3.0, 4.9, 1.8, 2); +INSERT into iris (a, b, c, d, label) values (6.4, 2.8, 5.6, 2.1, 2);INSERT into iris (a, b, c, d, label) values (7.2, 3.0, 5.8, 1.6, 2); +INSERT into iris (a, b, c, d, label) values (7.4, 2.8, 6.1, 1.9, 2);INSERT into iris (a, b, c, d, label) values (7.9, 3.8, 6.4, 2.0, 2); +INSERT into iris (a, b, c, d, label) values (6.4, 2.8, 5.6, 2.2, 2);INSERT into iris (a, b, c, d, label) values (6.3, 2.8, 5.1, 1.5, 2); +INSERT into iris (a, b, c, d, label) values (6.1, 2.6, 5.6, 1.4, 2);INSERT into iris (a, b, c, d, label) values (7.7, 3.0, 6.1, 2.3, 2); +INSERT into iris (a, b, c, d, label) values (6.3, 3.4, 5.6, 2.4, 2);INSERT into iris (a, b, c, d, label) values (6.4, 3.1, 5.5, 1.8, 2); +INSERT into iris (a, b, c, d, label) values (6.0, 3.0, 4.8, 1.8, 2);INSERT into iris (a, b, c, d, label) values (6.9, 3.1, 5.4, 2.1, 2); +INSERT into iris (a, b, c, d, label) values (6.7, 3.1, 5.6, 2.4, 2);INSERT into iris (a, b, c, d, label) values (6.9, 3.1, 5.1, 2.3, 2); +INSERT into iris (a, b, c, d, label) values (5.8, 2.7, 5.1, 1.9, 2);INSERT into iris (a, b, c, d, label) values (6.8, 3.2, 5.9, 2.3, 2); +INSERT into iris (a, b, c, d, label) values (6.7, 3.3, 5.7, 2.5, 2);INSERT into iris (a, b, c, d, label) values (6.7, 3.0, 5.2, 2.3, 2); +INSERT into iris (a, b, c, d, label) values (6.3, 2.5, 5.0, 1.9, 2);INSERT into iris (a, b, c, d, label) values (6.5, 3.0, 5.2, 2.0, 2); +INSERT into iris (a, b, c, d, label) values (6.2, 3.4, 5.4, 2.3, 2);INSERT into iris (a, b, c, d, label) values (5.9, 3.0, 5.1, 1.8, 2); + +drop table if exists iris_model_xgbc; +select MADLIB_SCHEMA.xgboost_sk_Classifier('iris', 'iris_model_xgbc', 'id', 'label', 'a,b,c,d', NULL, $${'booster': ['gbtree'], 'eta': (0.1, 0.9), 'max_depth': (5,1), 'objective': ('multi:softmax',)}$$); + +select id, train_timestamp, source_table, y_type, metrics, features, params from iris_model_xgbc; + +select assert(count(*) = 4, 'xgboost: xgboost parameter counts error') from iris_model_xgbc; + +select assert(metrics = 'evalution is OFF, no metrics', 'xgboost: xgboost metrics error') from iris_model_xgbc where id = 1; + +---------------- +drop table if exists iris_model_xgbc; +select MADLIB_SCHEMA.xgboost_sk_Classifier('iris', 'iris_model_xgbc', 'id', 'label', 'a,b,c,d', NULL, $${'booster': ['gbtree'], 'eta': (0.1, 0.9), 'max_depth': (5,1), 'objective': ('multi:softmax',)}$$, TRUE); + +select id, train_timestamp, source_table, y_type, metrics, features, params from iris_model_xgbc; +select assert(count(*) = 4, 'xgboost: xgboost parameter counts error') from iris_model_xgbc; +select assert(metrics <> 'evalution is OFF, no metrics', 'xgboost: xgboost metrics error') from iris_model_xgbc where id = 1; + +----------- +drop table if exists iris_xgbc_out; +select MADLIB_SCHEMA.xgboost_sk_predict('iris', 'iris_model_xgbc', 'iris_xgbc_out', 'id'); + +drop table if exists iris_model_xgbc; +select xgboost_sk_Classifier('iris', 'iris_model_xgbc', 'id', 'label', '*', NULL, $${'booster': ('gbtree',), 'eta': [0.1, 0.9], 'max_depth': (5,1), 'objective': ['multi:softmax']}$$); + +SELECT assert(count(*) = 4, 'xgboost: Xgboost model output has incorrect number of rows') +FROM (select train_timestamp, source_table, y_type, metrics, features, params, id from iris_model_xgbc) q; + +drop table if exists iris_xgbc_out; +select MADLIB_SCHEMA.xgboost_sk_predict('iris', 'iris_model_xgbc', 'iris_xgbc_out', 'id'); + +drop table if exists temp_iris; +create table temp_iris as select t1.id, prediction, label from iris as t1, iris_xgbc_out as t2 where t1.id = t2.id order by t1.id; + +select assert(count(*) >135, 'xgboost: xgboost prediction incorrect') +from (select * from temp_iris where prediction = label) q; + + +----------------------------------------------------------- binaryl classification ---------------------------------------------- + +drop table if exists xgbctable; +create table xgbctable(id serial, "X" float, Y float, label int); + +insert into xgbctable("X", y, label) values (1.349880, 3.503074, 0);insert into xgbctable("X", y, label) values (3.605580, 4.285024, 0); +insert into xgbctable("X", y, label) values (2.537560, 5.671145, 0);insert into xgbctable("X", y, label) values (2.596382, 4.219879, 0); +insert into xgbctable("X", y, label) values (3.043375, 3.425613, 0);insert into xgbctable("X", y, label) values (2.277542, 2.532572, 0); +insert into xgbctable("X", y, label) values (4.036647, 3.891047, 0);insert into xgbctable("X", y, label) values (3.814163, 3.737898, 0); +insert into xgbctable("X", y, label) values (2.674975, 4.340797, 0);insert into xgbctable("X", y, label) values (1.694112, 4.533308, 0); +insert into xgbctable("X", y, label) values (2.623954, 4.337177, 0);insert into xgbctable("X", y, label) values (3.412750, 6.994476, 0); +insert into xgbctable("X", y, label) values (3.562286, 3.827778, 0);insert into xgbctable("X", y, label) values (3.278937, 4.116019, 0); +insert into xgbctable("X", y, label) values (2.691412, 3.482750, 0);insert into xgbctable("X", y, label) values (1.947223, 3.734410, 0); +insert into xgbctable("X", y, label) values (1.376026, 3.232049, 0);insert into xgbctable("X", y, label) values (1.499600, 7.429014, 0); +insert into xgbctable("X", y, label) values (3.971720, 4.967350, 0);insert into xgbctable("X", y, label) values (2.984960, 3.780627, 0); +insert into xgbctable("X", y, label) values (3.493389, 3.693761, 0);insert into xgbctable("X", y, label) values (3.653253, 4.156218, 0); +insert into xgbctable("X", y, label) values (2.552306, 3.436848, 0);insert into xgbctable("X", y, label) values (3.395667, 2.731429, 0); +insert into xgbctable("X", y, label) values (4.356895, 3.604831, 0);insert into xgbctable("X", y, label) values (2.703829, 2.192765, 0); +insert into xgbctable("X", y, label) values (2.507021, 4.329131, 0);insert into xgbctable("X", y, label) values (5.437925, 4.798840, 0); +insert into xgbctable("X", y, label) values (4.275079, 4.472619, 0);insert into xgbctable("X", y, label) values (3.829493, 5.539909, 0); +insert into xgbctable("X", y, label) values (2.148925, 4.254559, 0);insert into xgbctable("X", y, label) values (2.798221, 3.916840, 0); +insert into xgbctable("X", y, label) values (3.388155, 3.060772, 0);insert into xgbctable("X", y, label) values (2.741393, 2.502708, 0); +insert into xgbctable("X", y, label) values (3.930997, 3.435667, 0);insert into xgbctable("X", y, label) values (2.641057, 4.070123, 0); +insert into xgbctable("X", y, label) values (1.631679, 5.890544, 0);insert into xgbctable("X", y, label) values (3.857039, 3.486820, 0); +insert into xgbctable("X", y, label) values (4.009723, 5.215680, 0);insert into xgbctable("X", y, label) values (4.815413, 3.742883, 0); +insert into xgbctable("X", y, label) values (2.010829, 3.793627, 0);insert into xgbctable("X", y, label) values (3.803213, 4.966541, 0); +insert into xgbctable("X", y, label) values (3.074533, 3.769696, 0);insert into xgbctable("X", y, label) values (3.305067, 3.941329, 0); +insert into xgbctable("X", y, label) values (4.255057, 2.714015, 0);insert into xgbctable("X", y, label) values (1.990719, 3.748694, 0); +insert into xgbctable("X", y, label) values (3.271186, 1.932663, 0);insert into xgbctable("X", y, label) values (5.712751, 4.925150, 0); +insert into xgbctable("X", y, label) values (4.436121, 1.918319, 0);insert into xgbctable("X", y, label) values (1.467938, 3.670151, 0); +insert into xgbctable("X", y, label) values (2.986207, 4.793286, 0);insert into xgbctable("X", y, label) values (2.599137, 2.807718, 0); +insert into xgbctable("X", y, label) values (3.108875, 2.852789, 0);insert into xgbctable("X", y, label) values (3.031777, 4.673723, 0); +insert into xgbctable("X", y, label) values (3.262225, 2.773082, 0);insert into xgbctable("X", y, label) values (2.022650, 4.607023, 0); +insert into xgbctable("X", y, label) values (3.147877, 3.796113, 0);insert into xgbctable("X", y, label) values (3.111487, 3.129382, 0); +insert into xgbctable("X", y, label) values (2.982505, 4.189220, 0);insert into xgbctable("X", y, label) values (2.928811, 4.102743, 0); +insert into xgbctable("X", y, label) values (3.121620, 5.194476, 0);insert into xgbctable("X", y, label) values (3.520649, 3.870720, 0); +insert into xgbctable("X", y, label) values (1.329847, 4.796737, 0);insert into xgbctable("X", y, label) values (4.355449, 3.361799, 0); +insert into xgbctable("X", y, label) values (3.125785, 4.588572, 0);insert into xgbctable("X", y, label) values (2.465354, 3.974886, 0); +insert into xgbctable("X", y, label) values (4.031982, 3.994294, 0);insert into xgbctable("X", y, label) values (2.649743, 4.096057, 0); +insert into xgbctable("X", y, label) values (3.613181, 3.801199, 0);insert into xgbctable("X", y, label) values (6.107873, 4.124081, 0); +insert into xgbctable("X", y, label) values (3.574605, 4.029697, 0);insert into xgbctable("X", y, label) values (2.426873, 2.782740, 0); +insert into xgbctable("X", y, label) values (2.159410, 3.899774, 0);insert into xgbctable("X", y, label) values (2.625650, 2.297274, 0); +insert into xgbctable("X", y, label) values (3.069545, 4.923757, 0);insert into xgbctable("X", y, label) values (2.950428, 2.858376, 0); +insert into xgbctable("X", y, label) values (3.601556, 2.587292, 0);insert into xgbctable("X", y, label) values (3.639818, 4.304443, 0); +insert into xgbctable("X", y, label) values (2.759995, 2.999176, 0);insert into xgbctable("X", y, label) values (1.826029, 3.564573, 0); +insert into xgbctable("X", y, label) values (4.016062, 3.730916, 0);insert into xgbctable("X", y, label) values (2.464892, 3.112348, 0); +insert into xgbctable("X", y, label) values (4.325916, 5.596980, 0);insert into xgbctable("X", y, label) values (2.131952, 4.396857, 0); +insert into xgbctable("X", y, label) values (3.473718, 4.673927, 0);insert into xgbctable("X", y, label) values (3.711892, 4.908759, 0); +insert into xgbctable("X", y, label) values (3.700971, 3.462295, 0);insert into xgbctable("X", y, label) values (3.125731, 5.828338, 0); +insert into xgbctable("X", y, label) values (3.957516, 5.104233, 0);insert into xgbctable("X", y, label) values (4.658009, 5.078593, 0); +insert into xgbctable("X", y, label) values (2.576429, 3.274793, 0);insert into xgbctable("X", y, label) values (3.160974, 3.466810, 0); +insert into xgbctable("X", y, label) values (2.613970, 2.838497, 0);insert into xgbctable("X", y, label) values (2.385694, 3.749695, 0); +insert into xgbctable("X", y, label) values (2.640589, 2.259807, 0);insert into xgbctable("X", y, label) values (4.589973, 2.776422, 0); +insert into xgbctable("X", y, label) values (1.875313, 3.571599, 0);insert into xgbctable("X", y, label) values (3.472463, 5.251267, 0); +insert into xgbctable("X", y, label) values (2.638684, 4.845147, 0);insert into xgbctable("X", y, label) values (3.405707, 2.662543, 0); + +insert into xgbctable("X", y, label) values (3.092827, -0.757907, 1);insert into xgbctable("X", y, label) values (1.982449, 0.781236, 1); +insert into xgbctable("X", y, label) values (1.880089, 3.638690, 1);insert into xgbctable("X", y, label) values (-0.143234, -0.823097, 1); +insert into xgbctable("X", y, label) values (1.913393, 0.717394, 1);insert into xgbctable("X", y, label) values (-0.406319, 1.495522, 1); +insert into xgbctable("X", y, label) values (1.982796, -0.043737, 1);insert into xgbctable("X", y, label) values (2.001371, 2.005737, 1); +insert into xgbctable("X", y, label) values (1.683994, 2.722670, 1);insert into xgbctable("X", y, label) values (1.201483, -0.544765, 1); +insert into xgbctable("X", y, label) values (1.054981, 0.857019, 1);insert into xgbctable("X", y, label) values (-1.137237, 0.769533, 1); +insert into xgbctable("X", y, label) values (0.526662, -0.076017, 1);insert into xgbctable("X", y, label) values (0.431452, 1.018968, 1); +insert into xgbctable("X", y, label) values (0.911789, -1.003597, 1);insert into xgbctable("X", y, label) values (0.271256, 0.749327, 1); +insert into xgbctable("X", y, label) values (-0.141832, -1.478830, 1);insert into xgbctable("X", y, label) values (0.508604, 0.648365, 1); +insert into xgbctable("X", y, label) values (1.903264, 0.034425, 1);insert into xgbctable("X", y, label) values (0.788896, 0.615347, 1); +insert into xgbctable("X", y, label) values (3.350388, -0.238897, 1);insert into xgbctable("X", y, label) values (-0.632393, 0.605239, 1); +insert into xgbctable("X", y, label) values (1.170017, 0.536877, 1);insert into xgbctable("X", y, label) values (1.381837, 0.453970, 1); +insert into xgbctable("X", y, label) values (0.238530, 0.234563, 1);insert into xgbctable("X", y, label) values (-0.619639, -0.739849, 1); +insert into xgbctable("X", y, label) values (1.782465, 2.802568, 1);insert into xgbctable("X", y, label) values (-1.690702, 0.989325, 1); +insert into xgbctable("X", y, label) values (0.109266, 2.755468, 1);insert into xgbctable("X", y, label) values (1.670044, 1.392086, 1); +insert into xgbctable("X", y, label) values (2.461095, -0.625544, 1);insert into xgbctable("X", y, label) values (1.580424, 1.225731, 1); +insert into xgbctable("X", y, label) values (0.949291, 0.709720, 1);insert into xgbctable("X", y, label) values (1.919173, 0.810358, 1); +insert into xgbctable("X", y, label) values (1.086786, 2.770693, 1);insert into xgbctable("X", y, label) values (0.955707, 1.763802, 1); +insert into xgbctable("X", y, label) values (1.689414, 1.062906, 1);insert into xgbctable("X", y, label) values (1.438960, 1.282794, 1); +insert into xgbctable("X", y, label) values (0.499648, 1.425253, 1);insert into xgbctable("X", y, label) values (0.712802, -0.080972, 1); +insert into xgbctable("X", y, label) values (1.501847, 1.318616, 1);insert into xgbctable("X", y, label) values (-0.718307, 0.644610, 1); +insert into xgbctable("X", y, label) values (-0.259394, 1.004450, 1);insert into xgbctable("X", y, label) values (1.822353, 1.461093, 1); +insert into xgbctable("X", y, label) values (0.065569, 2.134341, 1);insert into xgbctable("X", y, label) values (1.137744, 1.431270, 1); +insert into xgbctable("X", y, label) values (1.613366, -0.399205, 1);insert into xgbctable("X", y, label) values (2.780135, 0.523116, 1); +insert into xgbctable("X", y, label) values (1.641254, 0.189915, 1);insert into xgbctable("X", y, label) values (0.224779, 0.320547, 1); +insert into xgbctable("X", y, label) values (1.509881, 0.488949, 1);insert into xgbctable("X", y, label) values (2.742344, 1.078382, 1); +insert into xgbctable("X", y, label) values (0.015037, 0.906546, 1);insert into xgbctable("X", y, label) values (0.790595, 0.721075, 1); +insert into xgbctable("X", y, label) values (0.296409, 0.922898, 1);insert into xgbctable("X", y, label) values (1.116124, 2.510543, 1); +insert into xgbctable("X", y, label) values (1.104642, 1.043124, 1);insert into xgbctable("X", y, label) values (1.261002, 1.275877, 1); +insert into xgbctable("X", y, label) values (0.995230, 1.131128, 1);insert into xgbctable("X", y, label) values (0.500961, 1.053070, 1); +insert into xgbctable("X", y, label) values (0.753316, 1.587285, 1);insert into xgbctable("X", y, label) values (1.415908, 1.080478, 1); +insert into xgbctable("X", y, label) values (1.391515, 0.915593, 1);insert into xgbctable("X", y, label) values (1.309082, 0.667459, 1); +insert into xgbctable("X", y, label) values (0.622153, 2.885750, 1);insert into xgbctable("X", y, label) values (2.499712, 1.917895, 1); +insert into xgbctable("X", y, label) values (0.754728, 1.182577, 1);insert into xgbctable("X", y, label) values (1.653910, -0.448352, 1); +insert into xgbctable("X", y, label) values (2.263763, -0.022151, 1);insert into xgbctable("X", y, label) values (-0.289203, 2.267931, 1); +insert into xgbctable("X", y, label) values (2.117514, 0.852814, 1);insert into xgbctable("X", y, label) values (0.682405, 0.043468, 1); +insert into xgbctable("X", y, label) values (1.304387, 1.649660, 1);insert into xgbctable("X", y, label) values (2.775033, 0.608014, 1); +insert into xgbctable("X", y, label) values (0.695554, 1.631088, 1);insert into xgbctable("X", y, label) values (1.736803, 0.258182, 1); +insert into xgbctable("X", y, label) values (1.369698, 1.281675, 1);insert into xgbctable("X", y, label) values (-0.461257, 2.632165, 1); +insert into xgbctable("X", y, label) values (1.672385, 0.965322, 1);insert into xgbctable("X", y, label) values (0.527026, 0.849661, 1); +insert into xgbctable("X", y, label) values (1.583187, 0.823039, 1);insert into xgbctable("X", y, label) values (1.217844, 1.465343, 1); +insert into xgbctable("X", y, label) values (-0.190087, -0.190168, 1);insert into xgbctable("X", y, label) values (0.939329, -0.837490, 1); +insert into xgbctable("X", y, label) values (0.421842, -1.023412, 1);insert into xgbctable("X", y, label) values (1.015774, 0.422640, 1); +insert into xgbctable("X", y, label) values (0.036181, -1.399000, 1);insert into xgbctable("X", y, label) values (1.001890, 1.183682, 1); +insert into xgbctable("X", y, label) values (3.219715, 1.811829, 1);insert into xgbctable("X", y, label) values (2.270468, 1.717835, 1); +insert into xgbctable("X", y, label) values (0.354725, 0.918786, 1);insert into xgbctable("X", y, label) values (3.065704, 0.358424, 1); +insert into xgbctable("X", y, label) values (-0.609859, 0.152997, 1);insert into xgbctable("X", y, label) values (0.476704, 0.071017, 1); +insert into xgbctable("X", y, label) values (0.494553, 3.201871, 1);insert into xgbctable("X", y, label) values (3.430162, -0.834660, 1); +insert into xgbctable("X", y, label) values (2.542903, 1.831189, 1);insert into xgbctable("X", y, label) values (1.973389, 0.079485, 1); +insert into xgbctable("X", y, label) values (1.309454, 0.648461, 1);insert into xgbctable("X", y, label) values (0.841486, -0.177773, 1); + +drop table if exists xgbctable_model; +select MADLIB_SCHEMA.xgboost_sk_Classifier('xgbctable', 'xgbctable_model', 'id', 'label', '"X", Y', NULL, $${'booster': ('gbtree',), 'eta': (0.1), 'max_depth': 5, 'objective': ('binary:logistic',)}$$, TRUE); + +select id, train_timestamp, source_table, y_type, metrics, features, params from xgbctable_model; +select assert(count(*) = 1, 'xgboost: xgboost parameter counts error') from xgbctable_model; + +drop table if exists xgbctable_model; +select MADLIB_SCHEMA.xgboost_sk_Classifier('xgbctable', 'xgbctable_model', 'id', 'label', '*', 'id, label', $${'booster': ['gbtree'], 'eta': 0.1, 'max_depth': (5), 'objective': ['binary:logistic']}$$); +select assert(metrics = 'evalution is OFF, no metrics', 'xgboost: evalution error') from xgbctable_model; + + +drop table if exists xgbctable_output; +select MADLIB_SCHEMA.xgboost_sk_predict('xgbctable', 'xgbctable_model', 'xgbctable_output', 'id', TRUE); + +drop table if exists xgbctable_output_summary; +create table xgbctable_output_summary as select t1.id, prediction, label from xgbctable_output as t1, xgbctable as t2 where t1.id = t2.id order by t1.id; + +drop table if exists xgbctable_aur; +select MADLIB_SCHEMA.area_under_roc('xgbctable_output_summary', 'xgbctable_aur', 'prediction', 'label'); +select * from xgbctable_aur; +select assert(area_under_roc > 0.9, 'xgbctable AUR error') from xgbctable_aur; + +drop table if exists xgbctable_cm; +select MADLIB_SCHEMA.confusion_matrix('xgbctable_output_summary', 'xgbctable_cm', 'prediction', 'label'); +select * from xgbctable_cm; diff --git a/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_regress.sql_in b/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_regress.sql_in new file mode 100644 index 0000000..99e216a --- /dev/null +++ b/src/ports/postgres/modules/xgboost_gs/test/xgboost_sk_regress.sql_in @@ -0,0 +1,554 @@ +/* + * Test case for sklearn-api xgboost. + */ + +m4_include(`SQLCommon.m4') + +select xgboost_sk_regressor('usage'); +select xgboost_sk_regressor('help'); +select xgboost_sk_regressor('?'); +select xgboost_sk_regressor(''); + +DROP TABLE IF EXISTS boston; +CREATE TABLE boston ( +id integer NOT NULL, crim double precision, zn float8, Indus double precision, +chas double precision, nox float, rm float4, "Age" int, +dis DECIMAL(4), rad double precision, tax double precision, ptratio float, +b double precision, LstaT double precision, target float); + +INSERT INTO boston VALUES (2, 0.027310000000000001, 0, 7.0700000000000003, 0, 0.46899999999999997, 6.4210000000000003, 78.900000000000006, 4.9671000000000003, 2, 242, 17.800000000000001, 396.89999999999998, 9.1400000000000006, 21.600000000000001); +INSERT INTO boston VALUES (6, 0.029850000000000002, 0, 2.1800000000000002, 0, 0.45800000000000002, 6.4299999999999997, 58.700000000000003, 6.0621999999999998, 3, 222, 18.699999999999999, 394.12, 5.21, 28.699999999999999); +INSERT INTO boston VALUES (10, 0.17004, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.0039999999999996, 85.900000000000006, 6.5921000000000003, 5, 311, 15.199999999999999, 386.70999999999998, 17.100000000000001, 18.899999999999999); +INSERT INTO boston VALUES (14, 0.62975999999999999, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9489999999999998, 61.799999999999997, 4.7074999999999996, 4, 307, 21, 396.89999999999998, 8.2599999999999998, 20.399999999999999); +INSERT INTO boston VALUES (18, 0.78420000000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9900000000000002, 81.700000000000003, 4.2579000000000002, 4, 307, 21, 386.75, 14.67, 17.5); +INSERT INTO boston VALUES (22, 0.85204000000000002, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9649999999999999, 89.200000000000003, 4.0122999999999998, 4, 307, 21, 392.52999999999997, 13.83, 19.600000000000001); +INSERT INTO boston VALUES (26, 0.84053999999999995, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.5990000000000002, 85.700000000000003, 4.4546000000000001, 4, 307, 21, 303.42000000000002, 16.510000000000002, 13.9); +INSERT INTO boston VALUES (30, 1.0024500000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.6740000000000004, 87.299999999999997, 4.2389999999999999, 4, 307, 21, 380.23000000000002, 11.98, 21); +INSERT INTO boston VALUES (34, 1.1517200000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.7009999999999996, 95, 3.7871999999999999, 4, 307, 21, 358.76999999999998, 18.350000000000001, 13.1); +INSERT INTO boston VALUES (38, 0.080140000000000003, 0, 5.96, 0, 0.499, 5.8499999999999996, 41.5, 3.9342000000000001, 5, 279, 19.199999999999999, 396.89999999999998, 8.7699999999999996, 21); +INSERT INTO boston VALUES (42, 0.12744, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.7699999999999996, 2.8999999999999999, 5.7209000000000003, 3, 233, 17.899999999999999, 385.41000000000003, 4.8399999999999999, 26.600000000000001); +INSERT INTO boston VALUES (46, 0.17141999999999999, 0, 6.9100000000000001, 0, 0.44800000000000001, 5.6820000000000004, 33.799999999999997, 5.1003999999999996, 3, 233, 17.899999999999999, 396.89999999999998, 10.210000000000001, 19.300000000000001); +INSERT INTO boston VALUES (50, 0.21976999999999999, 0, 6.9100000000000001, 0, 0.44800000000000001, 5.6020000000000003, 62, 6.0876999999999999, 3, 233, 17.899999999999999, 396.89999999999998, 16.199999999999999, 19.399999999999999); +INSERT INTO boston VALUES (54, 0.04981, 21, 5.6399999999999997, 0, 0.439, 5.9980000000000002, 21.399999999999999, 6.8147000000000002, 4, 243, 16.800000000000001, 396.89999999999998, 8.4299999999999997, 23.399999999999999); +INSERT INTO boston VALUES (58, 0.014319999999999999, 100, 1.3200000000000001, 0, 0.41099999999999998, 6.8159999999999998, 40.5, 8.3247999999999998, 5, 256, 15.1, 392.89999999999998, 3.9500000000000002, 31.600000000000001); +INSERT INTO boston VALUES (62, 0.17171, 25, 5.1299999999999999, 0, 0.45300000000000001, 5.9660000000000002, 93.400000000000006, 6.8185000000000002, 8, 284, 19.699999999999999, 378.07999999999998, 14.44, 16); +INSERT INTO boston VALUES (66, 0.035839999999999997, 80, 3.3700000000000001, 0, 0.39800000000000002, 6.29, 17.800000000000001, 6.6115000000000004, 4, 337, 16.100000000000001, 396.89999999999998, 4.6699999999999999, 23.5); +INSERT INTO boston VALUES (70, 0.12816, 12.5, 6.0700000000000003, 0, 0.40899999999999997, 5.8849999999999998, 33, 6.4980000000000002, 4, 345, 18.899999999999999, 396.89999999999998, 8.7899999999999991, 20.899999999999999); +INSERT INTO boston VALUES (74, 0.19539000000000001, 0, 10.81, 0, 0.41299999999999998, 6.2450000000000001, 6.2000000000000002, 5.2873000000000001, 4, 305, 19.199999999999999, 377.17000000000002, 7.54, 23.399999999999999); +INSERT INTO boston VALUES (78, 0.087069999999999995, 0, 12.83, 0, 0.437, 6.1399999999999997, 45.799999999999997, 4.0904999999999996, 5, 398, 18.699999999999999, 386.95999999999998, 10.27, 20.800000000000001); +INSERT INTO boston VALUES (82, 0.04462, 25, 4.8600000000000003, 0, 0.42599999999999999, 6.6189999999999998, 70.400000000000006, 5.4006999999999996, 4, 281, 19, 395.63, 7.2199999999999998, 23.899999999999999); +INSERT INTO boston VALUES (86, 0.057349999999999998, 0, 4.4900000000000002, 0, 0.44900000000000001, 6.6299999999999999, 56.100000000000001, 4.4377000000000004, 3, 247, 18.5, 392.30000000000001, 6.5300000000000002, 26.600000000000001); +INSERT INTO boston VALUES (90, 0.053019999999999998, 0, 3.4100000000000001, 0, 0.48899999999999999, 7.0789999999999997, 63.100000000000001, 3.4144999999999999, 2, 270, 17.800000000000001, 396.06, 5.7000000000000002, 28.699999999999999); +INSERT INTO boston VALUES (94, 0.028750000000000001, 28, 15.039999999999999, 0, 0.46400000000000002, 6.2110000000000003, 28.899999999999999, 3.6659000000000002, 4, 270, 18.199999999999999, 396.32999999999998, 6.21, 25); +INSERT INTO boston VALUES (98, 0.12083000000000001, 0, 2.8900000000000001, 0, 0.44500000000000001, 8.0690000000000008, 76, 3.4952000000000001, 2, 276, 18, 396.89999999999998, 4.21, 38.700000000000003); +INSERT INTO boston VALUES (102, 0.11432, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.7809999999999997, 71.299999999999997, 2.8561000000000001, 5, 384, 20.899999999999999, 395.57999999999998, 7.6699999999999999, 26.5); +INSERT INTO boston VALUES (106, 0.13261999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 5.851, 96.700000000000003, 2.1069, 5, 384, 20.899999999999999, 394.05000000000001, 16.469999999999999, 19.5); +INSERT INTO boston VALUES (110, 0.26362999999999998, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.2290000000000001, 91.200000000000003, 2.5451000000000001, 5, 384, 20.899999999999999, 391.23000000000002, 15.550000000000001, 19.399999999999999); +INSERT INTO boston VALUES (114, 0.22212000000000001, 0, 10.01, 0, 0.54700000000000004, 6.0919999999999996, 95.400000000000006, 2.548, 6, 432, 17.800000000000001, 396.89999999999998, 17.09, 18.699999999999999); +INSERT INTO boston VALUES (118, 0.15098, 0, 10.01, 0, 0.54700000000000004, 6.0209999999999999, 82.599999999999994, 2.7473999999999998, 6, 432, 17.800000000000001, 394.50999999999999, 10.300000000000001, 19.199999999999999); +INSERT INTO boston VALUES (122, 0.071650000000000005, 0, 25.649999999999999, 0, 0.58099999999999996, 6.0039999999999996, 84.099999999999994, 2.1974, 2, 188, 19.100000000000001, 377.67000000000002, 14.27, 20.300000000000001); +INSERT INTO boston VALUES (126, 0.16902, 0, 25.649999999999999, 0, 0.58099999999999996, 5.9859999999999998, 88.400000000000006, 1.9928999999999999, 2, 188, 19.100000000000001, 385.01999999999998, 14.81, 21.399999999999999); +INSERT INTO boston VALUES (130, 0.88124999999999998, 0, 21.890000000000001, 0, 0.624, 5.6369999999999996, 94.700000000000003, 1.9799, 4, 437, 21.199999999999999, 396.89999999999998, 18.34, 14.300000000000001); +INSERT INTO boston VALUES (134, 0.32982, 0, 21.890000000000001, 0, 0.624, 5.8220000000000001, 95.400000000000006, 2.4699, 4, 437, 21.199999999999999, 388.69, 15.029999999999999, 18.399999999999999); +INSERT INTO boston VALUES (138, 0.35232999999999998, 0, 21.890000000000001, 0, 0.624, 6.4539999999999997, 98.400000000000006, 1.8498000000000001, 4, 437, 21.199999999999999, 394.07999999999998, 14.59, 17.100000000000001); +INSERT INTO boston VALUES (142, 1.6286400000000001, 0, 21.890000000000001, 0, 0.624, 5.0190000000000001, 100, 1.4394, 4, 437, 21.199999999999999, 396.89999999999998, 34.409999999999997, 14.4); +INSERT INTO boston VALUES (146, 2.37934, 0, 19.579999999999998, 0, 0.871, 6.1299999999999999, 100, 1.4191, 5, 403, 14.699999999999999, 172.91, 27.800000000000001, 13.800000000000001); +INSERT INTO boston VALUES (150, 2.7339699999999998, 0, 19.579999999999998, 0, 0.871, 5.5970000000000004, 94.900000000000006, 1.5257000000000001, 5, 403, 14.699999999999999, 351.85000000000002, 21.449999999999999, 15.4); +INSERT INTO boston VALUES (154, 2.1491799999999999, 0, 19.579999999999998, 0, 0.871, 5.7089999999999996, 98.5, 1.6232, 5, 403, 14.699999999999999, 261.94999999999999, 15.789999999999999, 19.399999999999999); +INSERT INTO boston VALUES (158, 1.2235799999999999, 0, 19.579999999999998, 0, 0.60499999999999998, 6.9429999999999996, 97.400000000000006, 1.8773, 5, 403, 14.699999999999999, 363.43000000000001, 4.5899999999999999, 41.299999999999997); +INSERT INTO boston VALUES (162, 1.46336, 0, 19.579999999999998, 0, 0.60499999999999998, 7.4889999999999999, 90.799999999999997, 1.9709000000000001, 5, 403, 14.699999999999999, 374.43000000000001, 1.73, 50); +INSERT INTO boston VALUES (166, 2.9239999999999999, 0, 19.579999999999998, 0, 0.60499999999999998, 6.101, 93, 2.2833999999999999, 5, 403, 14.699999999999999, 240.16, 9.8100000000000005, 25); +INSERT INTO boston VALUES (170, 2.4495300000000002, 0, 19.579999999999998, 0, 0.60499999999999998, 6.4020000000000001, 95.200000000000003, 2.2625000000000002, 5, 403, 14.699999999999999, 330.04000000000002, 11.32, 22.300000000000001); +INSERT INTO boston VALUES (174, 0.09178, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.4160000000000004, 84.099999999999994, 2.6463000000000001, 5, 296, 16.600000000000001, 395.5, 9.0399999999999991, 23.600000000000001); +INSERT INTO boston VALUES (178, 0.05425, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.3150000000000004, 73.400000000000006, 3.3174999999999999, 5, 296, 16.600000000000001, 395.60000000000002, 6.29, 24.600000000000001); +INSERT INTO boston VALUES (182, 0.068879999999999997, 0, 2.46, 0, 0.48799999999999999, 6.1440000000000001, 62.200000000000003, 2.5979000000000001, 3, 193, 17.800000000000001, 396.89999999999998, 9.4499999999999993, 36.200000000000003); +INSERT INTO boston VALUES (186, 0.060470000000000003, 0, 2.46, 0, 0.48799999999999999, 6.1529999999999996, 68.799999999999997, 3.2797000000000001, 3, 193, 17.800000000000001, 387.11000000000001, 13.15, 29.600000000000001); +INSERT INTO boston VALUES (190, 0.083699999999999997, 45, 3.4399999999999999, 0, 0.437, 7.1849999999999996, 38.899999999999999, 4.5667, 5, 398, 15.199999999999999, 396.89999999999998, 5.3899999999999997, 34.899999999999999); +INSERT INTO boston VALUES (194, 0.021870000000000001, 60, 2.9300000000000002, 0, 0.40100000000000002, 6.7999999999999998, 9.9000000000000004, 6.2195999999999998, 1, 265, 15.6, 393.37, 5.0300000000000002, 31.100000000000001); +INSERT INTO boston VALUES (198, 0.04666, 80, 1.52, 0, 0.40400000000000003, 7.1070000000000002, 36.600000000000001, 7.3090000000000002, 2, 329, 12.6, 354.31, 8.6099999999999994, 30.300000000000001); +INSERT INTO boston VALUES (202, 0.034450000000000001, 82.5, 2.0299999999999998, 0, 0.41499999999999998, 6.1619999999999999, 38.399999999999999, 6.2699999999999996, 2, 348, 14.699999999999999, 393.76999999999998, 7.4299999999999997, 24.100000000000001); +INSERT INTO boston VALUES (206, 0.13642000000000001, 0, 10.59, 0, 0.48899999999999999, 5.891, 22.300000000000001, 3.9453999999999998, 4, 277, 18.600000000000001, 396.89999999999998, 10.869999999999999, 22.600000000000001); +INSERT INTO boston VALUES (210, 0.43570999999999999, 0, 10.59, 1, 0.48899999999999999, 5.3440000000000003, 100, 3.875, 4, 277, 18.600000000000001, 396.89999999999998, 23.09, 20); +INSERT INTO boston VALUES (214, 0.14052000000000001, 0, 10.59, 0, 0.48899999999999999, 6.375, 32.299999999999997, 3.9453999999999998, 4, 277, 18.600000000000001, 385.81, 9.3800000000000008, 28.100000000000001); +INSERT INTO boston VALUES (218, 0.070129999999999998, 0, 13.890000000000001, 0, 0.55000000000000004, 6.6420000000000003, 85.099999999999994, 3.4211, 5, 276, 16.399999999999999, 392.77999999999997, 9.6899999999999995, 28.699999999999999); +INSERT INTO boston VALUES (222, 0.40771000000000002, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.1639999999999997, 91.299999999999997, 3.048, 8, 307, 17.399999999999999, 395.24000000000001, 21.460000000000001, 21.699999999999999); +INSERT INTO boston VALUES (226, 0.52693000000000001, 0, 6.2000000000000002, 0, 0.504, 8.7249999999999996, 83, 2.8944000000000001, 8, 307, 17.399999999999999, 382, 4.6299999999999999, 50); +INSERT INTO boston VALUES (230, 0.44178000000000001, 0, 6.2000000000000002, 0, 0.504, 6.5519999999999996, 21.399999999999999, 3.3751000000000002, 8, 307, 17.399999999999999, 380.33999999999997, 3.7599999999999998, 31.5); +INSERT INTO boston VALUES (1, 0.0063200000000000001, 18, 2.3100000000000001, 0, 0.53800000000000003, 6.5750000000000002, 65.200000000000003, 4.0899999999999999, 1, 296, 15.300000000000001, 396.89999999999998, 4.9800000000000004, 24); +INSERT INTO boston VALUES (5, 0.06905, 0, 2.1800000000000002, 0, 0.45800000000000002, 7.1470000000000002, 54.200000000000003, 6.0621999999999998, 3, 222, 18.699999999999999, 396.89999999999998, 5.3300000000000001, 36.200000000000003); +INSERT INTO boston VALUES (9, 0.21124000000000001, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 5.6310000000000002, 100, 6.0820999999999996, 5, 311, 15.199999999999999, 386.63, 29.93, 16.5); +INSERT INTO boston VALUES (13, 0.093780000000000002, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 5.8890000000000002, 39, 5.4508999999999999, 5, 311, 15.199999999999999, 390.5, 15.710000000000001, 21.699999999999999); +INSERT INTO boston VALUES (17, 1.05393, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9349999999999996, 29.300000000000001, 4.4985999999999997, 4, 307, 21, 386.85000000000002, 6.5800000000000001, 23.100000000000001); +INSERT INTO boston VALUES (21, 1.25179, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.5700000000000003, 98.099999999999994, 3.7978999999999998, 4, 307, 21, 376.56999999999999, 21.02, 13.6); +INSERT INTO boston VALUES (25, 0.75026000000000004, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9240000000000004, 94.099999999999994, 4.3996000000000004, 4, 307, 21, 394.32999999999998, 16.300000000000001, 15.6); +INSERT INTO boston VALUES (29, 0.77298999999999995, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.4950000000000001, 94.400000000000006, 4.4546999999999999, 4, 307, 21, 387.94, 12.800000000000001, 18.399999999999999); +INSERT INTO boston VALUES (33, 1.3879900000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.9500000000000002, 82, 3.9900000000000002, 4, 307, 21, 232.59999999999999, 27.710000000000001, 13.199999999999999); +INSERT INTO boston VALUES (37, 0.097439999999999999, 0, 5.96, 0, 0.499, 5.8410000000000002, 61.399999999999999, 3.3778999999999999, 5, 279, 19.199999999999999, 377.56, 11.41, 20); +INSERT INTO boston VALUES (41, 0.033590000000000002, 75, 2.9500000000000002, 0, 0.42799999999999999, 7.024, 15.800000000000001, 5.4010999999999996, 3, 252, 18.300000000000001, 395.62, 1.98, 34.899999999999999); +INSERT INTO boston VALUES (45, 0.12268999999999999, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.069, 40, 5.7209000000000003, 3, 233, 17.899999999999999, 389.38999999999999, 9.5500000000000007, 21.199999999999999); +INSERT INTO boston VALUES (49, 0.25386999999999998, 0, 6.9100000000000001, 0, 0.44800000000000001, 5.399, 95.299999999999997, 5.8700000000000001, 3, 233, 17.899999999999999, 396.89999999999998, 30.809999999999999, 14.4); +INSERT INTO boston VALUES (53, 0.053600000000000002, 21, 5.6399999999999997, 0, 0.439, 6.5110000000000001, 21.100000000000001, 6.8147000000000002, 4, 243, 16.800000000000001, 396.89999999999998, 5.2800000000000002, 25); +INSERT INTO boston VALUES (57, 0.020549999999999999, 85, 0.73999999999999999, 0, 0.40999999999999998, 6.383, 35.700000000000003, 9.1875999999999998, 2, 313, 17.300000000000001, 396.89999999999998, 5.7699999999999996, 24.699999999999999); +INSERT INTO boston VALUES (61, 0.14932000000000001, 25, 5.1299999999999999, 0, 0.45300000000000001, 5.7409999999999997, 66.200000000000003, 7.2253999999999996, 8, 284, 19.699999999999999, 395.11000000000001, 13.15, 18.699999999999999); +INSERT INTO boston VALUES (65, 0.01951, 17.5, 1.3799999999999999, 0, 0.41610000000000003, 7.1040000000000001, 59.5, 9.2228999999999992, 3, 216, 18.600000000000001, 393.24000000000001, 8.0500000000000007, 33); +INSERT INTO boston VALUES (69, 0.13553999999999999, 12.5, 6.0700000000000003, 0, 0.40899999999999997, 5.5940000000000003, 36.799999999999997, 6.4980000000000002, 4, 345, 18.899999999999999, 396.89999999999998, 13.09, 17.399999999999999); +INSERT INTO boston VALUES (73, 0.091639999999999999, 0, 10.81, 0, 0.41299999999999998, 6.0650000000000004, 7.7999999999999998, 5.2873000000000001, 4, 305, 19.199999999999999, 390.91000000000003, 5.5199999999999996, 22.800000000000001); +INSERT INTO boston VALUES (77, 0.10153, 0, 12.83, 0, 0.437, 6.2789999999999999, 74.5, 4.0522, 5, 398, 18.699999999999999, 373.66000000000003, 11.970000000000001, 20); +INSERT INTO boston VALUES (81, 0.04113, 25, 4.8600000000000003, 0, 0.42599999999999999, 6.7270000000000003, 33.5, 5.4006999999999996, 4, 281, 19, 396.89999999999998, 5.29, 28); +INSERT INTO boston VALUES (85, 0.050590000000000003, 0, 4.4900000000000002, 0, 0.44900000000000001, 6.3890000000000002, 48, 4.7793999999999999, 3, 247, 18.5, 396.89999999999998, 9.6199999999999992, 23.899999999999999); +INSERT INTO boston VALUES (89, 0.056599999999999998, 0, 3.4100000000000001, 0, 0.48899999999999999, 7.0069999999999997, 86.299999999999997, 3.4217, 2, 270, 17.800000000000001, 396.89999999999998, 5.5, 23.600000000000001); +INSERT INTO boston VALUES (93, 0.042029999999999998, 28, 15.039999999999999, 0, 0.46400000000000002, 6.4420000000000002, 53.600000000000001, 3.6659000000000002, 4, 270, 18.199999999999999, 395.00999999999999, 8.1600000000000001, 22.899999999999999); +INSERT INTO boston VALUES (97, 0.11504, 0, 2.8900000000000001, 0, 0.44500000000000001, 6.1630000000000003, 69.599999999999994, 3.4952000000000001, 2, 276, 18, 391.82999999999998, 11.34, 21.399999999999999); +INSERT INTO boston VALUES (101, 0.14865999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.7270000000000003, 79.900000000000006, 2.7778, 5, 384, 20.899999999999999, 394.75999999999999, 9.4199999999999999, 27.5); +INSERT INTO boston VALUES (105, 0.1396, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.1669999999999998, 90, 2.4209999999999998, 5, 384, 20.899999999999999, 392.69, 12.33, 20.100000000000001); +INSERT INTO boston VALUES (109, 0.12801999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.4740000000000002, 97.099999999999994, 2.4329000000000001, 5, 384, 20.899999999999999, 395.24000000000001, 12.27, 19.800000000000001); +INSERT INTO boston VALUES (113, 0.12329, 0, 10.01, 0, 0.54700000000000004, 5.9130000000000003, 92.900000000000006, 2.3534000000000002, 6, 432, 17.800000000000001, 394.94999999999999, 16.210000000000001, 18.800000000000001); +INSERT INTO boston VALUES (117, 0.13158, 0, 10.01, 0, 0.54700000000000004, 6.1760000000000002, 72.5, 2.7301000000000002, 6, 432, 17.800000000000001, 393.30000000000001, 12.039999999999999, 21.199999999999999); +INSERT INTO boston VALUES (121, 0.068989999999999996, 0, 25.649999999999999, 0, 0.58099999999999996, 5.8700000000000001, 69.700000000000003, 2.2576999999999998, 2, 188, 19.100000000000001, 389.14999999999998, 14.369999999999999, 22); +INSERT INTO boston VALUES (125, 0.098489999999999994, 0, 25.649999999999999, 0, 0.58099999999999996, 5.8789999999999996, 95.799999999999997, 2.0063, 2, 188, 19.100000000000001, 379.38, 17.579999999999998, 18.800000000000001); +INSERT INTO boston VALUES (129, 0.32543, 0, 21.890000000000001, 0, 0.624, 6.431, 98.799999999999997, 1.8125, 4, 437, 21.199999999999999, 396.89999999999998, 15.390000000000001, 18); +INSERT INTO boston VALUES (133, 0.59004999999999996, 0, 21.890000000000001, 0, 0.624, 6.3719999999999999, 97.900000000000006, 2.3273999999999999, 4, 437, 21.199999999999999, 385.75999999999999, 11.119999999999999, 23); +INSERT INTO boston VALUES (137, 0.32263999999999998, 0, 21.890000000000001, 0, 0.624, 5.9420000000000002, 93.5, 1.9669000000000001, 4, 437, 21.199999999999999, 378.25, 16.899999999999999, 17.399999999999999); +INSERT INTO boston VALUES (141, 0.29089999999999999, 0, 21.890000000000001, 0, 0.624, 6.1740000000000004, 93.599999999999994, 1.6119000000000001, 4, 437, 21.199999999999999, 388.07999999999998, 24.16, 14); +INSERT INTO boston VALUES (145, 2.7797399999999999, 0, 19.579999999999998, 0, 0.871, 4.9029999999999996, 97.799999999999997, 1.3459000000000001, 5, 403, 14.699999999999999, 396.89999999999998, 29.289999999999999, 11.800000000000001); +INSERT INTO boston VALUES (149, 2.3309899999999999, 0, 19.579999999999998, 0, 0.871, 5.1859999999999999, 93.799999999999997, 1.5296000000000001, 5, 403, 14.699999999999999, 356.99000000000001, 28.32, 17.800000000000001); +INSERT INTO boston VALUES (153, 1.1265799999999999, 0, 19.579999999999998, 1, 0.871, 5.0119999999999996, 88, 1.6102000000000001, 5, 403, 14.699999999999999, 343.27999999999997, 12.119999999999999, 15.300000000000001); +INSERT INTO boston VALUES (157, 2.4466800000000002, 0, 19.579999999999998, 0, 0.871, 5.2720000000000002, 94, 1.7363999999999999, 5, 403, 14.699999999999999, 88.629999999999995, 16.140000000000001, 13.1); +INSERT INTO boston VALUES (161, 1.27346, 0, 19.579999999999998, 1, 0.60499999999999998, 6.25, 92.599999999999994, 1.7984, 5, 403, 14.699999999999999, 338.92000000000002, 5.5, 27); +INSERT INTO boston VALUES (165, 2.2423600000000001, 0, 19.579999999999998, 0, 0.60499999999999998, 5.8540000000000001, 91.799999999999997, 2.4220000000000002, 5, 403, 14.699999999999999, 395.11000000000001, 11.640000000000001, 22.699999999999999); +INSERT INTO boston VALUES (169, 2.3003999999999998, 0, 19.579999999999998, 0, 0.60499999999999998, 6.319, 96.099999999999994, 2.1000000000000001, 5, 403, 14.699999999999999, 297.08999999999997, 11.1, 23.800000000000001); +INSERT INTO boston VALUES (173, 0.13914000000000001, 0, 4.0499999999999998, 0, 0.51000000000000001, 5.5720000000000001, 88.5, 2.5960999999999999, 5, 296, 16.600000000000001, 396.89999999999998, 14.69, 23.100000000000001); +INSERT INTO boston VALUES (177, 0.070220000000000005, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.0199999999999996, 47.200000000000003, 3.5548999999999999, 5, 296, 16.600000000000001, 393.23000000000002, 10.109999999999999, 23.199999999999999); +INSERT INTO boston VALUES (181, 0.065879999999999994, 0, 2.46, 0, 0.48799999999999999, 7.7649999999999997, 83.299999999999997, 2.7410000000000001, 3, 193, 17.800000000000001, 395.56, 7.5599999999999996, 39.799999999999997); +INSERT INTO boston VALUES (185, 0.083080000000000001, 0, 2.46, 0, 0.48799999999999999, 5.6040000000000001, 89.799999999999997, 2.9878999999999998, 3, 193, 17.800000000000001, 391, 13.98, 26.399999999999999); +INSERT INTO boston VALUES (189, 0.12579000000000001, 45, 3.4399999999999999, 0, 0.437, 6.556, 29.100000000000001, 4.5667, 5, 398, 15.199999999999999, 382.83999999999997, 4.5599999999999996, 29.800000000000001); +INSERT INTO boston VALUES (193, 0.086639999999999995, 45, 3.4399999999999999, 0, 0.437, 7.1779999999999999, 26.300000000000001, 6.4798, 5, 398, 15.199999999999999, 390.49000000000001, 2.8700000000000001, 36.399999999999999); +INSERT INTO boston VALUES (197, 0.04011, 80, 1.52, 0, 0.40400000000000003, 7.2869999999999999, 34.100000000000001, 7.3090000000000002, 2, 329, 12.6, 396.89999999999998, 4.0800000000000001, 33.299999999999997); +INSERT INTO boston VALUES (201, 0.017780000000000001, 95, 1.47, 0, 0.40300000000000002, 7.1349999999999998, 13.9, 7.6534000000000004, 3, 402, 17, 384.30000000000001, 4.4500000000000002, 32.899999999999999); +INSERT INTO boston VALUES (205, 0.02009, 95, 2.6800000000000002, 0, 0.41610000000000003, 8.0340000000000007, 31.899999999999999, 5.1180000000000003, 4, 224, 14.699999999999999, 390.55000000000001, 2.8799999999999999, 50); +INSERT INTO boston VALUES (209, 0.13586999999999999, 0, 10.59, 1, 0.48899999999999999, 6.0640000000000001, 59.100000000000001, 4.2392000000000003, 4, 277, 18.600000000000001, 381.31999999999999, 14.66, 24.399999999999999); +INSERT INTO boston VALUES (213, 0.21718999999999999, 0, 10.59, 1, 0.48899999999999999, 5.8070000000000004, 53.799999999999997, 3.6526000000000001, 4, 277, 18.600000000000001, 390.94, 16.030000000000001, 22.399999999999999); +INSERT INTO boston VALUES (217, 0.045600000000000002, 0, 13.890000000000001, 1, 0.55000000000000004, 5.8879999999999999, 56, 3.1120999999999999, 5, 276, 16.399999999999999, 392.80000000000001, 13.51, 23.300000000000001); +INSERT INTO boston VALUES (221, 0.35809000000000002, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.9509999999999996, 88.5, 2.8616999999999999, 8, 307, 17.399999999999999, 391.69999999999999, 9.7100000000000009, 26.699999999999999); +INSERT INTO boston VALUES (225, 0.31533, 0, 6.2000000000000002, 0, 0.504, 8.266, 78.299999999999997, 2.8944000000000001, 8, 307, 17.399999999999999, 385.05000000000001, 4.1399999999999997, 44.799999999999997); +INSERT INTO boston VALUES (229, 0.29819000000000001, 0, 6.2000000000000002, 0, 0.504, 7.6859999999999999, 17, 3.3751000000000002, 8, 307, 17.399999999999999, 377.50999999999999, 3.9199999999999999, 46.700000000000003); +INSERT INTO boston VALUES (234, 0.33146999999999999, 0, 6.2000000000000002, 0, 0.50700000000000001, 8.2469999999999999, 70.400000000000006, 3.6518999999999999, 8, 307, 17.399999999999999, 378.94999999999999, 3.9500000000000002, 48.299999999999997); +INSERT INTO boston VALUES (238, 0.51183000000000001, 0, 6.2000000000000002, 0, 0.50700000000000001, 7.3579999999999997, 71.599999999999994, 4.1479999999999997, 8, 307, 17.399999999999999, 390.06999999999999, 4.7300000000000004, 31.5); +INSERT INTO boston VALUES (242, 0.10612000000000001, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.0949999999999998, 65.099999999999994, 6.3361000000000001, 6, 300, 16.600000000000001, 394.62, 12.4, 20.100000000000001); +INSERT INTO boston VALUES (246, 0.19133, 22, 5.8600000000000003, 0, 0.43099999999999999, 5.6050000000000004, 70.200000000000003, 7.9549000000000003, 7, 330, 19.100000000000001, 389.13, 18.460000000000001, 18.5); +INSERT INTO boston VALUES (250, 0.19073000000000001, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.718, 17.5, 7.8265000000000002, 7, 330, 19.100000000000001, 393.74000000000001, 6.5599999999999996, 26.199999999999999); +INSERT INTO boston VALUES (254, 0.36893999999999999, 22, 5.8600000000000003, 0, 0.43099999999999999, 8.2590000000000003, 8.4000000000000004, 8.9067000000000007, 7, 330, 19.100000000000001, 396.89999999999998, 3.54, 42.799999999999997); +INSERT INTO boston VALUES (257, 0.01538, 90, 3.75, 0, 0.39400000000000002, 7.4539999999999997, 34.200000000000003, 6.3361000000000001, 3, 244, 15.9, 386.33999999999997, 3.1099999999999999, 44); +INSERT INTO boston VALUES (261, 0.54010999999999998, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.2030000000000003, 81.799999999999997, 2.1120999999999999, 5, 264, 13, 392.80000000000001, 9.5899999999999999, 33.799999999999997); +INSERT INTO boston VALUES (265, 0.55006999999999995, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.2060000000000004, 91.599999999999994, 1.9300999999999999, 5, 264, 13, 387.88999999999999, 8.0999999999999996, 36.5); +INSERT INTO boston VALUES (269, 0.54049999999999998, 20, 3.9700000000000002, 0, 0.57499999999999996, 7.4699999999999998, 52.600000000000001, 2.8719999999999999, 5, 264, 13, 390.30000000000001, 3.1600000000000001, 43.5); +INSERT INTO boston VALUES (273, 0.11459999999999999, 20, 6.96, 0, 0.46400000000000002, 6.5380000000000003, 58.700000000000003, 3.9175, 3, 223, 18.600000000000001, 394.95999999999998, 7.7300000000000004, 24.399999999999999); +INSERT INTO boston VALUES (277, 0.10469000000000001, 40, 6.4100000000000001, 1, 0.44700000000000001, 7.2670000000000003, 49, 4.7872000000000003, 4, 254, 17.600000000000001, 389.25, 6.0499999999999998, 33.200000000000003); +INSERT INTO boston VALUES (281, 0.035779999999999999, 20, 3.3300000000000001, 0, 0.44290000000000002, 7.8200000000000003, 64.5, 4.6947000000000001, 5, 216, 14.9, 387.31, 3.7599999999999998, 45.399999999999999); +INSERT INTO boston VALUES (285, 0.0090600000000000003, 90, 2.9700000000000002, 0, 0.40000000000000002, 7.0880000000000001, 20.800000000000001, 7.3072999999999997, 1, 285, 15.300000000000001, 394.72000000000003, 7.8499999999999996, 32.200000000000003); +INSERT INTO boston VALUES (289, 0.045900000000000003, 52.5, 5.3200000000000003, 0, 0.40500000000000003, 6.3150000000000004, 45.600000000000001, 7.3171999999999997, 6, 293, 16.600000000000001, 396.89999999999998, 7.5999999999999996, 22.300000000000001); +INSERT INTO boston VALUES (293, 0.036150000000000002, 80, 4.9500000000000002, 0, 0.41099999999999998, 6.6299999999999999, 23.399999999999999, 5.1166999999999998, 4, 245, 19.199999999999999, 396.89999999999998, 4.7000000000000002, 27.899999999999999); +INSERT INTO boston VALUES (297, 0.053719999999999997, 0, 13.92, 0, 0.437, 6.5490000000000004, 51, 5.9603999999999999, 4, 289, 16, 392.85000000000002, 7.3899999999999997, 27.100000000000001); +INSERT INTO boston VALUES (301, 0.044170000000000001, 70, 2.2400000000000002, 0, 0.40000000000000002, 6.8710000000000004, 47.399999999999999, 7.8277999999999999, 5, 358, 14.800000000000001, 390.86000000000001, 6.0700000000000003, 24.800000000000001); +INSERT INTO boston VALUES (305, 0.055149999999999998, 33, 2.1800000000000002, 0, 0.47199999999999998, 7.2359999999999998, 41.100000000000001, 4.0220000000000002, 7, 222, 18.399999999999999, 393.68000000000001, 6.9299999999999997, 36.100000000000001); +INSERT INTO boston VALUES (309, 0.49297999999999997, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.6349999999999998, 82.5, 3.3174999999999999, 4, 304, 18.399999999999999, 396.89999999999998, 4.54, 22.800000000000001); +INSERT INTO boston VALUES (313, 0.26168999999999998, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.0229999999999997, 90.400000000000006, 2.8340000000000001, 4, 304, 18.399999999999999, 396.30000000000001, 11.720000000000001, 19.399999999999999); +INSERT INTO boston VALUES (317, 0.31827, 0, 9.9000000000000004, 0, 0.54400000000000004, 5.9139999999999997, 83.200000000000003, 3.9986000000000002, 4, 304, 18.399999999999999, 390.69999999999999, 18.329999999999998, 17.800000000000001); +INSERT INTO boston VALUES (321, 0.1676, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.4260000000000002, 52.299999999999997, 4.5404, 5, 287, 19.600000000000001, 396.89999999999998, 7.2000000000000002, 23.800000000000001); +INSERT INTO boston VALUES (325, 0.34109, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.415, 40.100000000000001, 4.7210999999999999, 5, 287, 19.600000000000001, 396.89999999999998, 6.1200000000000001, 25); +INSERT INTO boston VALUES (329, 0.066170000000000007, 0, 3.2400000000000002, 0, 0.46000000000000002, 5.8680000000000003, 25.800000000000001, 5.2145999999999999, 4, 430, 16.899999999999999, 382.44, 9.9700000000000006, 19.300000000000001); +INSERT INTO boston VALUES (333, 0.034660000000000003, 35, 6.0599999999999996, 0, 0.43790000000000001, 6.0309999999999997, 23.300000000000001, 6.6406999999999998, 1, 304, 16.899999999999999, 362.25, 7.8300000000000001, 19.399999999999999); +INSERT INTO boston VALUES (337, 0.034270000000000002, 0, 5.1900000000000004, 0, 0.51500000000000001, 5.8689999999999998, 46.299999999999997, 5.2310999999999996, 5, 224, 20.199999999999999, 396.89999999999998, 9.8000000000000007, 19.5); +INSERT INTO boston VALUES (341, 0.061510000000000002, 0, 5.1900000000000004, 0, 0.51500000000000001, 5.968, 58.5, 4.8121999999999998, 5, 224, 20.199999999999999, 396.89999999999998, 9.2899999999999991, 18.699999999999999); +INSERT INTO boston VALUES (345, 0.03049, 55, 3.7799999999999998, 0, 0.48399999999999999, 6.8739999999999997, 28.100000000000001, 6.4653999999999998, 5, 370, 17.600000000000001, 387.97000000000003, 4.6100000000000003, 31.199999999999999); +INSERT INTO boston VALUES (349, 0.015010000000000001, 80, 2.0099999999999998, 0, 0.435, 6.6349999999999998, 29.699999999999999, 8.3439999999999994, 4, 280, 17, 390.94, 5.9900000000000002, 24.5); +INSERT INTO boston VALUES (353, 0.072440000000000004, 60, 1.6899999999999999, 0, 0.41099999999999998, 5.8840000000000003, 18.5, 10.7103, 4, 411, 18.300000000000001, 392.32999999999998, 7.79, 18.600000000000001); +INSERT INTO boston VALUES (357, 8.9829600000000003, 0, 18.100000000000001, 1, 0.77000000000000002, 6.2119999999999997, 97.400000000000006, 2.1221999999999999, 24, 666, 20.199999999999999, 377.73000000000002, 17.600000000000001, 17.800000000000001); +INSERT INTO boston VALUES (361, 4.5419200000000002, 0, 18.100000000000001, 0, 0.77000000000000002, 6.3979999999999997, 88, 2.5182000000000002, 24, 666, 20.199999999999999, 374.56, 7.79, 25); +INSERT INTO boston VALUES (365, 3.4742799999999998, 0, 18.100000000000001, 1, 0.71799999999999997, 8.7799999999999994, 82.900000000000006, 1.9047000000000001, 24, 666, 20.199999999999999, 354.55000000000001, 5.29, 21.899999999999999); +INSERT INTO boston VALUES (369, 4.8982200000000002, 0, 18.100000000000001, 0, 0.63100000000000001, 4.9699999999999998, 100, 1.3325, 24, 666, 20.199999999999999, 375.51999999999998, 3.2599999999999998, 50); +INSERT INTO boston VALUES (373, 8.2672500000000007, 0, 18.100000000000001, 1, 0.66800000000000004, 5.875, 89.599999999999994, 1.1295999999999999, 24, 666, 20.199999999999999, 347.88, 8.8800000000000008, 50); +INSERT INTO boston VALUES (377, 15.288, 0, 18.100000000000001, 0, 0.67100000000000004, 6.649, 93.299999999999997, 1.3449, 24, 666, 20.199999999999999, 363.01999999999998, 23.239999999999998, 13.9); +INSERT INTO boston VALUES (381, 88.976200000000006, 0, 18.100000000000001, 0, 0.67100000000000004, 6.968, 91.900000000000006, 1.4165000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 17.210000000000001, 10.4); +INSERT INTO boston VALUES (385, 20.084900000000001, 0, 18.100000000000001, 0, 0.69999999999999996, 4.3680000000000003, 91.200000000000003, 1.4395, 24, 666, 20.199999999999999, 285.82999999999998, 30.629999999999999, 8.8000000000000007); +INSERT INTO boston VALUES (389, 14.3337, 0, 18.100000000000001, 0, 0.69999999999999996, 4.8799999999999999, 100, 1.5894999999999999, 24, 666, 20.199999999999999, 372.92000000000002, 30.620000000000001, 10.199999999999999); +INSERT INTO boston VALUES (393, 11.5779, 0, 18.100000000000001, 0, 0.69999999999999996, 5.0359999999999996, 97, 1.77, 24, 666, 20.199999999999999, 396.89999999999998, 25.68, 9.6999999999999993); +INSERT INTO boston VALUES (397, 5.8720499999999998, 0, 18.100000000000001, 0, 0.69299999999999995, 6.4050000000000002, 96, 1.6768000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 19.370000000000001, 12.5); +INSERT INTO boston VALUES (401, 25.046099999999999, 0, 18.100000000000001, 0, 0.69299999999999995, 5.9870000000000001, 100, 1.5888, 24, 666, 20.199999999999999, 396.89999999999998, 26.77, 5.5999999999999996); +INSERT INTO boston VALUES (405, 41.529200000000003, 0, 18.100000000000001, 0, 0.69299999999999995, 5.5309999999999997, 85.400000000000006, 1.6073999999999999, 24, 666, 20.199999999999999, 329.45999999999998, 27.379999999999999, 8.5); +INSERT INTO boston VALUES (409, 7.4038899999999996, 0, 18.100000000000001, 0, 0.59699999999999998, 5.617, 97.900000000000006, 1.4547000000000001, 24, 666, 20.199999999999999, 314.63999999999999, 26.399999999999999, 17.199999999999999); +INSERT INTO boston VALUES (413, 18.811, 0, 18.100000000000001, 0, 0.59699999999999998, 4.6280000000000001, 100, 1.5539000000000001, 24, 666, 20.199999999999999, 28.789999999999999, 34.369999999999997, 17.899999999999999); +INSERT INTO boston VALUES (417, 10.834199999999999, 0, 18.100000000000001, 0, 0.67900000000000005, 6.782, 90.799999999999997, 1.8194999999999999, 24, 666, 20.199999999999999, 21.57, 25.789999999999999, 7.5); +INSERT INTO boston VALUES (421, 11.087400000000001, 0, 18.100000000000001, 0, 0.71799999999999997, 6.4109999999999996, 100, 1.8589, 24, 666, 20.199999999999999, 318.75, 15.02, 16.699999999999999); +INSERT INTO boston VALUES (425, 8.7921200000000006, 0, 18.100000000000001, 0, 0.58399999999999996, 5.5650000000000004, 70.599999999999994, 2.0634999999999999, 24, 666, 20.199999999999999, 3.6499999999999999, 17.16, 11.699999999999999); +INSERT INTO boston VALUES (429, 7.3671100000000003, 0, 18.100000000000001, 0, 0.67900000000000005, 6.1929999999999996, 78.099999999999994, 1.9356, 24, 666, 20.199999999999999, 96.730000000000004, 21.52, 11); +INSERT INTO boston VALUES (433, 6.4440499999999998, 0, 18.100000000000001, 0, 0.58399999999999996, 6.4249999999999998, 74.799999999999997, 2.2004000000000001, 24, 666, 20.199999999999999, 97.950000000000003, 12.029999999999999, 16.100000000000001); +INSERT INTO boston VALUES (437, 14.4208, 0, 18.100000000000001, 0, 0.73999999999999999, 6.4610000000000003, 93.299999999999997, 2.0026000000000002, 24, 666, 20.199999999999999, 27.489999999999998, 18.050000000000001, 9.5999999999999996); +INSERT INTO boston VALUES (441, 22.051100000000002, 0, 18.100000000000001, 0, 0.73999999999999999, 5.8179999999999996, 92.400000000000006, 1.8662000000000001, 24, 666, 20.199999999999999, 391.44999999999999, 22.109999999999999, 10.5); +INSERT INTO boston VALUES (445, 12.802300000000001, 0, 18.100000000000001, 0, 0.73999999999999999, 5.8540000000000001, 96.599999999999994, 1.8956, 24, 666, 20.199999999999999, 240.52000000000001, 23.789999999999999, 10.800000000000001); +INSERT INTO boston VALUES (449, 9.3290900000000008, 0, 18.100000000000001, 0, 0.71299999999999997, 6.1849999999999996, 98.700000000000003, 2.2616000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 18.129999999999999, 14.1); +INSERT INTO boston VALUES (453, 5.0901699999999996, 0, 18.100000000000001, 0, 0.71299999999999997, 6.2969999999999997, 91.799999999999997, 2.3681999999999999, 24, 666, 20.199999999999999, 385.08999999999997, 17.27, 16.100000000000001); +INSERT INTO boston VALUES (457, 4.6688299999999998, 0, 18.100000000000001, 0, 0.71299999999999997, 5.976, 87.900000000000006, 2.5806, 24, 666, 20.199999999999999, 10.48, 19.010000000000002, 12.699999999999999); +INSERT INTO boston VALUES (461, 4.8121299999999998, 0, 18.100000000000001, 0, 0.71299999999999997, 6.7009999999999996, 90, 2.5975000000000001, 24, 666, 20.199999999999999, 255.22999999999999, 16.420000000000002, 16.399999999999999); +INSERT INTO boston VALUES (3, 0.027289999999999998, 0, 7.0700000000000003, 0, 0.46899999999999997, 7.1849999999999996, 61.100000000000001, 4.9671000000000003, 2, 242, 17.800000000000001, 392.82999999999998, 4.0300000000000002, 34.700000000000003); +INSERT INTO boston VALUES (7, 0.088289999999999993, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.0119999999999996, 66.599999999999994, 5.5605000000000002, 5, 311, 15.199999999999999, 395.60000000000002, 12.43, 22.899999999999999); +INSERT INTO boston VALUES (11, 0.22489000000000001, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.3769999999999998, 94.299999999999997, 6.3467000000000002, 5, 311, 15.199999999999999, 392.51999999999998, 20.449999999999999, 15); +INSERT INTO boston VALUES (15, 0.63795999999999997, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.0960000000000001, 84.5, 4.4619, 4, 307, 21, 380.01999999999998, 10.26, 18.199999999999999); +INSERT INTO boston VALUES (19, 0.80271000000000003, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.4560000000000004, 36.600000000000001, 3.7965, 4, 307, 21, 288.99000000000001, 11.69, 20.199999999999999); +INSERT INTO boston VALUES (23, 1.23247, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.1420000000000003, 91.700000000000003, 3.9769000000000001, 4, 307, 21, 396.89999999999998, 18.719999999999999, 15.199999999999999); +INSERT INTO boston VALUES (27, 0.67191000000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.8129999999999997, 90.299999999999997, 4.6820000000000004, 4, 307, 21, 376.88, 14.81, 16.600000000000001); +INSERT INTO boston VALUES (31, 1.1308100000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.7130000000000001, 94.099999999999994, 4.2329999999999997, 4, 307, 21, 360.17000000000002, 22.600000000000001, 12.699999999999999); +INSERT INTO boston VALUES (35, 1.6128199999999999, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.0960000000000001, 96.900000000000006, 3.7597999999999998, 4, 307, 21, 248.31, 20.34, 13.5); +INSERT INTO boston VALUES (39, 0.17505000000000001, 0, 5.96, 0, 0.499, 5.9660000000000002, 30.199999999999999, 3.8473000000000002, 5, 279, 19.199999999999999, 393.43000000000001, 10.130000000000001, 24.699999999999999); +INSERT INTO boston VALUES (43, 0.14149999999999999, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.1689999999999996, 6.5999999999999996, 5.7209000000000003, 3, 233, 17.899999999999999, 383.37, 5.8099999999999996, 25.300000000000001); +INSERT INTO boston VALUES (47, 0.18836, 0, 6.9100000000000001, 0, 0.44800000000000001, 5.7859999999999996, 33.299999999999997, 5.1003999999999996, 3, 233, 17.899999999999999, 396.89999999999998, 14.15, 20); +INSERT INTO boston VALUES (51, 0.088730000000000003, 21, 5.6399999999999997, 0, 0.439, 5.9630000000000001, 45.700000000000003, 6.8147000000000002, 4, 243, 16.800000000000001, 395.56, 13.449999999999999, 19.699999999999999); +INSERT INTO boston VALUES (55, 0.013599999999999999, 75, 4, 0, 0.40999999999999998, 5.8879999999999999, 47.600000000000001, 7.3197000000000001, 3, 469, 21.100000000000001, 396.89999999999998, 14.800000000000001, 18.899999999999999); +INSERT INTO boston VALUES (59, 0.15445, 25, 5.1299999999999999, 0, 0.45300000000000001, 6.1449999999999996, 29.199999999999999, 7.8148, 8, 284, 19.699999999999999, 390.68000000000001, 6.8600000000000003, 23.300000000000001); +INSERT INTO boston VALUES (63, 0.11027000000000001, 25, 5.1299999999999999, 0, 0.45300000000000001, 6.4560000000000004, 67.799999999999997, 7.2255000000000003, 8, 284, 19.699999999999999, 396.89999999999998, 6.7300000000000004, 22.199999999999999); +INSERT INTO boston VALUES (67, 0.043790000000000003, 80, 3.3700000000000001, 0, 0.39800000000000002, 5.7869999999999999, 31.100000000000001, 6.6115000000000004, 4, 337, 16.100000000000001, 396.89999999999998, 10.24, 19.399999999999999); +INSERT INTO boston VALUES (71, 0.088260000000000005, 0, 10.81, 0, 0.41299999999999998, 6.4169999999999998, 6.5999999999999996, 5.2873000000000001, 4, 305, 19.199999999999999, 383.73000000000002, 6.7199999999999998, 24.199999999999999); +INSERT INTO boston VALUES (75, 0.078960000000000002, 0, 12.83, 0, 0.437, 6.2729999999999997, 6, 4.2515000000000001, 5, 398, 18.699999999999999, 394.92000000000002, 6.7800000000000002, 24.100000000000001); +INSERT INTO boston VALUES (79, 0.056460000000000003, 0, 12.83, 0, 0.437, 6.2320000000000002, 53.700000000000003, 5.0141, 5, 398, 18.699999999999999, 386.39999999999998, 12.34, 21.199999999999999); +INSERT INTO boston VALUES (83, 0.036589999999999998, 25, 4.8600000000000003, 0, 0.42599999999999999, 6.3019999999999996, 32.200000000000003, 5.4006999999999996, 4, 281, 19, 396.89999999999998, 6.7199999999999998, 24.800000000000001); +INSERT INTO boston VALUES (87, 0.051880000000000003, 0, 4.4900000000000002, 0, 0.44900000000000001, 6.0149999999999997, 45.100000000000001, 4.4272, 3, 247, 18.5, 395.99000000000001, 12.859999999999999, 22.5); +INSERT INTO boston VALUES (91, 0.04684, 0, 3.4100000000000001, 0, 0.48899999999999999, 6.4169999999999998, 66.099999999999994, 3.0922999999999998, 2, 270, 17.800000000000001, 392.18000000000001, 8.8100000000000005, 22.600000000000001); +INSERT INTO boston VALUES (95, 0.042939999999999999, 28, 15.039999999999999, 0, 0.46400000000000002, 6.2489999999999997, 77.299999999999997, 3.6150000000000002, 4, 270, 18.199999999999999, 396.89999999999998, 10.59, 20.600000000000001); +INSERT INTO boston VALUES (99, 0.081869999999999998, 0, 2.8900000000000001, 0, 0.44500000000000001, 7.8200000000000003, 36.899999999999999, 3.4952000000000001, 2, 276, 18, 393.52999999999997, 3.5699999999999998, 43.799999999999997); +INSERT INTO boston VALUES (103, 0.22875999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.4050000000000002, 85.400000000000006, 2.7147000000000001, 5, 384, 20.899999999999999, 70.799999999999997, 10.630000000000001, 18.600000000000001); +INSERT INTO boston VALUES (107, 0.17119999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 5.8360000000000003, 91.900000000000006, 2.2109999999999999, 5, 384, 20.899999999999999, 395.67000000000002, 18.66, 19.5); +INSERT INTO boston VALUES (111, 0.10793, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.1950000000000003, 54.399999999999999, 2.7778, 5, 384, 20.899999999999999, 393.49000000000001, 13, 21.699999999999999); +INSERT INTO boston VALUES (115, 0.14230999999999999, 0, 10.01, 0, 0.54700000000000004, 6.2539999999999996, 84.200000000000003, 2.2565, 6, 432, 17.800000000000001, 388.74000000000001, 10.449999999999999, 18.5); +INSERT INTO boston VALUES (119, 0.13058, 0, 10.01, 0, 0.54700000000000004, 5.8719999999999999, 73.099999999999994, 2.4775, 6, 432, 17.800000000000001, 338.63, 15.369999999999999, 20.399999999999999); +INSERT INTO boston VALUES (123, 0.092990000000000003, 0, 25.649999999999999, 0, 0.58099999999999996, 5.9610000000000003, 92.900000000000006, 2.0869, 2, 188, 19.100000000000001, 378.08999999999997, 17.93, 20.5); +INSERT INTO boston VALUES (127, 0.38735000000000003, 0, 25.649999999999999, 0, 0.58099999999999996, 5.6130000000000004, 95.599999999999994, 1.7572000000000001, 2, 188, 19.100000000000001, 359.29000000000002, 27.260000000000002, 15.699999999999999); +INSERT INTO boston VALUES (131, 0.34005999999999997, 0, 21.890000000000001, 0, 0.624, 6.4580000000000002, 98.900000000000006, 2.1185, 4, 437, 21.199999999999999, 395.04000000000002, 12.6, 19.199999999999999); +INSERT INTO boston VALUES (135, 0.97616999999999998, 0, 21.890000000000001, 0, 0.624, 5.7569999999999997, 98.400000000000006, 2.3460000000000001, 4, 437, 21.199999999999999, 262.75999999999999, 17.309999999999999, 15.6); +INSERT INTO boston VALUES (139, 0.24979999999999999, 0, 21.890000000000001, 0, 0.624, 5.8570000000000002, 98.200000000000003, 1.6686000000000001, 4, 437, 21.199999999999999, 392.04000000000002, 21.32, 13.300000000000001); +INSERT INTO boston VALUES (143, 3.3210500000000001, 0, 19.579999999999998, 1, 0.871, 5.4029999999999996, 100, 1.3216000000000001, 5, 403, 14.699999999999999, 396.89999999999998, 26.82, 13.4); +INSERT INTO boston VALUES (147, 2.1550500000000001, 0, 19.579999999999998, 0, 0.871, 5.6280000000000001, 100, 1.5165999999999999, 5, 403, 14.699999999999999, 169.27000000000001, 16.649999999999999, 15.6); +INSERT INTO boston VALUES (151, 1.6566000000000001, 0, 19.579999999999998, 0, 0.871, 6.1219999999999999, 97.299999999999997, 1.6180000000000001, 5, 403, 14.699999999999999, 372.80000000000001, 14.1, 21.5); +INSERT INTO boston VALUES (155, 1.4138500000000001, 0, 19.579999999999998, 1, 0.871, 6.1289999999999996, 96, 1.7494000000000001, 5, 403, 14.699999999999999, 321.01999999999998, 15.119999999999999, 17); +INSERT INTO boston VALUES (159, 1.34284, 0, 19.579999999999998, 0, 0.60499999999999998, 6.0659999999999998, 100, 1.7573000000000001, 5, 403, 14.699999999999999, 353.88999999999999, 6.4299999999999997, 24.300000000000001); +INSERT INTO boston VALUES (163, 1.8337699999999999, 0, 19.579999999999998, 1, 0.60499999999999998, 7.8019999999999996, 98.200000000000003, 2.0407000000000002, 5, 403, 14.699999999999999, 389.61000000000001, 1.9199999999999999, 50); +INSERT INTO boston VALUES (167, 2.0101900000000001, 0, 19.579999999999998, 0, 0.60499999999999998, 7.9290000000000003, 96.200000000000003, 2.0459000000000001, 5, 403, 14.699999999999999, 369.30000000000001, 3.7000000000000002, 50); +INSERT INTO boston VALUES (171, 1.2074199999999999, 0, 19.579999999999998, 0, 0.60499999999999998, 5.875, 94.599999999999994, 2.4258999999999999, 5, 403, 14.699999999999999, 292.29000000000002, 14.43, 17.399999999999999); +INSERT INTO boston VALUES (175, 0.084470000000000003, 0, 4.0499999999999998, 0, 0.51000000000000001, 5.859, 68.700000000000003, 2.7019000000000002, 5, 296, 16.600000000000001, 393.23000000000002, 9.6400000000000006, 22.600000000000001); +INSERT INTO boston VALUES (179, 0.066420000000000007, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.8600000000000003, 74.400000000000006, 2.9152999999999998, 5, 296, 16.600000000000001, 391.26999999999998, 6.9199999999999999, 29.899999999999999); +INSERT INTO boston VALUES (183, 0.09103, 0, 2.46, 0, 0.48799999999999999, 7.1550000000000002, 92.200000000000003, 2.7006000000000001, 3, 193, 17.800000000000001, 394.12, 4.8200000000000003, 37.899999999999999); +INSERT INTO boston VALUES (187, 0.05602, 0, 2.46, 0, 0.48799999999999999, 7.8310000000000004, 53.600000000000001, 3.1991999999999998, 3, 193, 17.800000000000001, 392.63, 4.4500000000000002, 50); +INSERT INTO boston VALUES (191, 0.090679999999999997, 45, 3.4399999999999999, 0, 0.437, 6.9509999999999996, 21.5, 6.4798, 5, 398, 15.199999999999999, 377.68000000000001, 5.0999999999999996, 37); +INSERT INTO boston VALUES (195, 0.01439, 60, 2.9300000000000002, 0, 0.40100000000000002, 6.6040000000000001, 18.800000000000001, 6.2195999999999998, 1, 265, 15.6, 376.69999999999999, 4.3799999999999999, 29.100000000000001); +INSERT INTO boston VALUES (199, 0.037679999999999998, 80, 1.52, 0, 0.40400000000000003, 7.274, 38.299999999999997, 7.3090000000000002, 2, 329, 12.6, 392.19999999999999, 6.6200000000000001, 34.600000000000001); +INSERT INTO boston VALUES (203, 0.021770000000000001, 82.5, 2.0299999999999998, 0, 0.41499999999999998, 7.6100000000000003, 15.699999999999999, 6.2699999999999996, 2, 348, 14.699999999999999, 395.38, 3.1099999999999999, 42.299999999999997); +INSERT INTO boston VALUES (207, 0.22969000000000001, 0, 10.59, 0, 0.48899999999999999, 6.3259999999999996, 52.5, 4.3548999999999998, 4, 277, 18.600000000000001, 394.87, 10.970000000000001, 24.399999999999999); +INSERT INTO boston VALUES (211, 0.17446, 0, 10.59, 1, 0.48899999999999999, 5.96, 92.099999999999994, 3.8771, 4, 277, 18.600000000000001, 393.25, 17.27, 21.699999999999999); +INSERT INTO boston VALUES (215, 0.28954999999999997, 0, 10.59, 0, 0.48899999999999999, 5.4119999999999999, 9.8000000000000007, 3.5874999999999999, 4, 277, 18.600000000000001, 348.93000000000001, 29.550000000000001, 23.699999999999999); +INSERT INTO boston VALUES (219, 0.11069, 0, 13.890000000000001, 1, 0.55000000000000004, 5.9509999999999996, 93.799999999999997, 2.8893, 5, 276, 16.399999999999999, 396.89999999999998, 17.920000000000002, 21.5); +INSERT INTO boston VALUES (223, 0.62356, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.8789999999999996, 77.700000000000003, 3.2721, 8, 307, 17.399999999999999, 390.38999999999999, 9.9299999999999997, 27.5); +INSERT INTO boston VALUES (227, 0.38213999999999998, 0, 6.2000000000000002, 0, 0.504, 8.0399999999999991, 86.5, 3.2157, 8, 307, 17.399999999999999, 387.38, 3.1299999999999999, 37.600000000000001); +INSERT INTO boston VALUES (231, 0.53700000000000003, 0, 6.2000000000000002, 0, 0.504, 5.9809999999999999, 68.099999999999994, 3.6715, 8, 307, 17.399999999999999, 378.35000000000002, 11.65, 24.300000000000001); +INSERT INTO boston VALUES (4, 0.032370000000000003, 0, 2.1800000000000002, 0, 0.45800000000000002, 6.9980000000000002, 45.799999999999997, 6.0621999999999998, 3, 222, 18.699999999999999, 394.63, 2.9399999999999999, 33.399999999999999); +INSERT INTO boston VALUES (8, 0.14455000000000001, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.1719999999999997, 96.099999999999994, 5.9504999999999999, 5, 311, 15.199999999999999, 396.89999999999998, 19.149999999999999, 27.100000000000001); +INSERT INTO boston VALUES (12, 0.11747, 12.5, 7.8700000000000001, 0, 0.52400000000000002, 6.0090000000000003, 82.900000000000006, 6.2267000000000001, 5, 311, 15.199999999999999, 396.89999999999998, 13.27, 18.899999999999999); +INSERT INTO boston VALUES (16, 0.62739, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.8339999999999996, 56.5, 4.4985999999999997, 4, 307, 21, 395.62, 8.4700000000000006, 19.899999999999999); +INSERT INTO boston VALUES (20, 0.7258, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.7270000000000003, 69.5, 3.7965, 4, 307, 21, 390.94999999999999, 11.279999999999999, 18.199999999999999); +INSERT INTO boston VALUES (24, 0.98843000000000003, 0, 8.1400000000000006, 0, 0.53800000000000003, 5.8129999999999997, 100, 4.0952000000000002, 4, 307, 21, 394.54000000000002, 19.879999999999999, 14.5); +INSERT INTO boston VALUES (28, 0.95577000000000001, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.0469999999999997, 88.799999999999997, 4.4534000000000002, 4, 307, 21, 306.38, 17.280000000000001, 14.800000000000001); +INSERT INTO boston VALUES (32, 1.3547199999999999, 0, 8.1400000000000006, 0, 0.53800000000000003, 6.0720000000000001, 100, 4.1749999999999998, 4, 307, 21, 376.73000000000002, 13.039999999999999, 14.5); +INSERT INTO boston VALUES (36, 0.064170000000000005, 0, 5.96, 0, 0.499, 5.9329999999999998, 68.200000000000003, 3.3603000000000001, 5, 279, 19.199999999999999, 396.89999999999998, 9.6799999999999997, 18.899999999999999); +INSERT INTO boston VALUES (40, 0.027629999999999998, 75, 2.9500000000000002, 0, 0.42799999999999999, 6.5949999999999998, 21.800000000000001, 5.4010999999999996, 3, 252, 18.300000000000001, 395.63, 4.3200000000000003, 30.800000000000001); +INSERT INTO boston VALUES (44, 0.15936, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.2110000000000003, 6.5, 5.7209000000000003, 3, 233, 17.899999999999999, 394.45999999999998, 7.4400000000000004, 24.699999999999999); +INSERT INTO boston VALUES (48, 0.22927, 0, 6.9100000000000001, 0, 0.44800000000000001, 6.0300000000000002, 85.5, 5.6894, 3, 233, 17.899999999999999, 392.74000000000001, 18.800000000000001, 16.600000000000001); +INSERT INTO boston VALUES (52, 0.043369999999999999, 21, 5.6399999999999997, 0, 0.439, 6.1150000000000002, 63, 6.8147000000000002, 4, 243, 16.800000000000001, 393.97000000000003, 9.4299999999999997, 20.5); +INSERT INTO boston VALUES (56, 0.01311, 90, 1.22, 0, 0.40300000000000002, 7.2489999999999997, 21.899999999999999, 8.6966000000000001, 5, 226, 17.899999999999999, 395.93000000000001, 4.8099999999999996, 35.399999999999999); +INSERT INTO boston VALUES (60, 0.10328, 25, 5.1299999999999999, 0, 0.45300000000000001, 5.9269999999999996, 47.200000000000003, 6.9320000000000004, 8, 284, 19.699999999999999, 396.89999999999998, 9.2200000000000006, 19.600000000000001); +INSERT INTO boston VALUES (64, 0.1265, 25, 5.1299999999999999, 0, 0.45300000000000001, 6.7619999999999996, 43.399999999999999, 7.9809000000000001, 8, 284, 19.699999999999999, 395.57999999999998, 9.5, 25); +INSERT INTO boston VALUES (68, 0.057889999999999997, 12.5, 6.0700000000000003, 0, 0.40899999999999997, 5.8780000000000001, 21.399999999999999, 6.4980000000000002, 4, 345, 18.899999999999999, 396.20999999999998, 8.0999999999999996, 22); +INSERT INTO boston VALUES (72, 0.15876000000000001, 0, 10.81, 0, 0.41299999999999998, 5.9610000000000003, 17.5, 5.2873000000000001, 4, 305, 19.199999999999999, 376.94, 9.8800000000000008, 21.699999999999999); +INSERT INTO boston VALUES (76, 0.095119999999999996, 0, 12.83, 0, 0.437, 6.2859999999999996, 45, 4.5026000000000002, 5, 398, 18.699999999999999, 383.23000000000002, 8.9399999999999995, 21.399999999999999); +INSERT INTO boston VALUES (80, 0.08387, 0, 12.83, 0, 0.437, 5.8739999999999997, 36.600000000000001, 4.5026000000000002, 5, 398, 18.699999999999999, 396.06, 9.0999999999999996, 20.300000000000001); +INSERT INTO boston VALUES (84, 0.03551, 25, 4.8600000000000003, 0, 0.42599999999999999, 6.1669999999999998, 46.700000000000003, 5.4006999999999996, 4, 281, 19, 390.63999999999999, 7.5099999999999998, 22.899999999999999); +INSERT INTO boston VALUES (88, 0.071510000000000004, 0, 4.4900000000000002, 0, 0.44900000000000001, 6.1210000000000004, 56.799999999999997, 3.7475999999999998, 3, 247, 18.5, 395.14999999999998, 8.4399999999999995, 22.199999999999999); +INSERT INTO boston VALUES (92, 0.039320000000000001, 0, 3.4100000000000001, 0, 0.48899999999999999, 6.4050000000000002, 73.900000000000006, 3.0920999999999998, 2, 270, 17.800000000000001, 393.55000000000001, 8.1999999999999993, 22); +INSERT INTO boston VALUES (96, 0.12204, 0, 2.8900000000000001, 0, 0.44500000000000001, 6.625, 57.799999999999997, 3.4952000000000001, 2, 276, 18, 357.98000000000002, 6.6500000000000004, 28.399999999999999); +INSERT INTO boston VALUES (100, 0.068599999999999994, 0, 2.8900000000000001, 0, 0.44500000000000001, 7.4160000000000004, 62.5, 3.4952000000000001, 2, 276, 18, 396.89999999999998, 6.1900000000000004, 33.200000000000003); +INSERT INTO boston VALUES (104, 0.21160999999999999, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.1369999999999996, 87.400000000000006, 2.7147000000000001, 5, 384, 20.899999999999999, 394.47000000000003, 13.44, 19.300000000000001); +INSERT INTO boston VALUES (108, 0.13117000000000001, 0, 8.5600000000000005, 0, 0.52000000000000002, 6.1269999999999998, 85.200000000000003, 2.1223999999999998, 5, 384, 20.899999999999999, 387.69, 14.09, 20.399999999999999); +INSERT INTO boston VALUES (112, 0.10084, 0, 10.01, 0, 0.54700000000000004, 6.7149999999999999, 81.599999999999994, 2.6775000000000002, 6, 432, 17.800000000000001, 395.58999999999997, 10.16, 22.800000000000001); +INSERT INTO boston VALUES (116, 0.17133999999999999, 0, 10.01, 0, 0.54700000000000004, 5.9279999999999999, 88.200000000000003, 2.4630999999999998, 6, 432, 17.800000000000001, 344.91000000000003, 15.76, 18.300000000000001); +INSERT INTO boston VALUES (120, 0.14476, 0, 10.01, 0, 0.54700000000000004, 5.7309999999999999, 65.200000000000003, 2.7591999999999999, 6, 432, 17.800000000000001, 391.5, 13.609999999999999, 19.300000000000001); +INSERT INTO boston VALUES (124, 0.15038000000000001, 0, 25.649999999999999, 0, 0.58099999999999996, 5.8559999999999999, 97, 1.9443999999999999, 2, 188, 19.100000000000001, 370.31, 25.41, 17.300000000000001); +INSERT INTO boston VALUES (128, 0.25914999999999999, 0, 21.890000000000001, 0, 0.624, 5.6929999999999996, 96, 1.7883, 4, 437, 21.199999999999999, 392.11000000000001, 17.190000000000001, 16.199999999999999); +INSERT INTO boston VALUES (132, 1.1929399999999999, 0, 21.890000000000001, 0, 0.624, 6.3259999999999996, 97.700000000000003, 2.2709999999999999, 4, 437, 21.199999999999999, 396.89999999999998, 12.26, 19.600000000000001); +INSERT INTO boston VALUES (136, 0.55778000000000005, 0, 21.890000000000001, 0, 0.624, 6.335, 98.200000000000003, 2.1107, 4, 437, 21.199999999999999, 394.67000000000002, 16.960000000000001, 18.100000000000001); +INSERT INTO boston VALUES (140, 0.54452, 0, 21.890000000000001, 0, 0.624, 6.1509999999999998, 97.900000000000006, 1.6687000000000001, 4, 437, 21.199999999999999, 396.89999999999998, 18.460000000000001, 17.800000000000001); +INSERT INTO boston VALUES (144, 4.0974000000000004, 0, 19.579999999999998, 0, 0.871, 5.468, 100, 1.4117999999999999, 5, 403, 14.699999999999999, 396.89999999999998, 26.420000000000002, 15.6); +INSERT INTO boston VALUES (148, 2.3686199999999999, 0, 19.579999999999998, 0, 0.871, 4.9260000000000002, 95.700000000000003, 1.4608000000000001, 5, 403, 14.699999999999999, 391.70999999999998, 29.530000000000001, 14.6); +INSERT INTO boston VALUES (152, 1.4963200000000001, 0, 19.579999999999998, 0, 0.871, 5.4039999999999999, 100, 1.5915999999999999, 5, 403, 14.699999999999999, 341.60000000000002, 13.279999999999999, 19.600000000000001); +INSERT INTO boston VALUES (156, 3.5350100000000002, 0, 19.579999999999998, 1, 0.871, 6.1520000000000001, 82.599999999999994, 1.7455000000000001, 5, 403, 14.699999999999999, 88.010000000000005, 15.02, 15.6); +INSERT INTO boston VALUES (160, 1.42502, 0, 19.579999999999998, 0, 0.871, 6.5099999999999998, 100, 1.7659, 5, 403, 14.699999999999999, 364.31, 7.3899999999999997, 23.300000000000001); +INSERT INTO boston VALUES (164, 1.51902, 0, 19.579999999999998, 1, 0.60499999999999998, 8.375, 93.900000000000006, 2.1619999999999999, 5, 403, 14.699999999999999, 388.44999999999999, 3.3199999999999998, 50); +INSERT INTO boston VALUES (168, 1.8002800000000001, 0, 19.579999999999998, 0, 0.60499999999999998, 5.8769999999999998, 79.200000000000003, 2.4258999999999999, 5, 403, 14.699999999999999, 227.61000000000001, 12.140000000000001, 23.800000000000001); +INSERT INTO boston VALUES (172, 2.3138999999999998, 0, 19.579999999999998, 0, 0.60499999999999998, 5.8799999999999999, 97.299999999999997, 2.3887, 5, 403, 14.699999999999999, 348.13, 12.029999999999999, 19.100000000000001); +INSERT INTO boston VALUES (176, 0.066640000000000005, 0, 4.0499999999999998, 0, 0.51000000000000001, 6.5460000000000003, 33.100000000000001, 3.1322999999999999, 5, 296, 16.600000000000001, 390.95999999999998, 5.3300000000000001, 29.399999999999999); +INSERT INTO boston VALUES (180, 0.057799999999999997, 0, 2.46, 0, 0.48799999999999999, 6.9800000000000004, 58.399999999999999, 2.8290000000000002, 3, 193, 17.800000000000001, 396.89999999999998, 5.04, 37.200000000000003); +INSERT INTO boston VALUES (184, 0.10008, 0, 2.46, 0, 0.48799999999999999, 6.5629999999999997, 95.599999999999994, 2.847, 3, 193, 17.800000000000001, 396.89999999999998, 5.6799999999999997, 32.5); +INSERT INTO boston VALUES (188, 0.078750000000000001, 45, 3.4399999999999999, 0, 0.437, 6.782, 41.100000000000001, 3.7886000000000002, 5, 398, 15.199999999999999, 393.87, 6.6799999999999997, 32); +INSERT INTO boston VALUES (192, 0.069110000000000005, 45, 3.4399999999999999, 0, 0.437, 6.7389999999999999, 30.800000000000001, 6.4798, 5, 398, 15.199999999999999, 389.70999999999998, 4.6900000000000004, 30.5); +INSERT INTO boston VALUES (196, 0.013809999999999999, 80, 0.46000000000000002, 0, 0.42199999999999999, 7.875, 32, 5.6483999999999996, 4, 255, 14.4, 394.23000000000002, 2.9700000000000002, 50); +INSERT INTO boston VALUES (200, 0.0315, 95, 1.47, 0, 0.40300000000000002, 6.9749999999999996, 15.300000000000001, 7.6534000000000004, 3, 402, 17, 396.89999999999998, 4.5599999999999996, 34.899999999999999); +INSERT INTO boston VALUES (204, 0.035099999999999999, 95, 2.6800000000000002, 0, 0.41610000000000003, 7.8529999999999998, 33.200000000000003, 5.1180000000000003, 4, 224, 14.699999999999999, 392.77999999999997, 3.8100000000000001, 48.5); +INSERT INTO boston VALUES (208, 0.25198999999999999, 0, 10.59, 0, 0.48899999999999999, 5.7830000000000004, 72.700000000000003, 4.3548999999999998, 4, 277, 18.600000000000001, 389.43000000000001, 18.059999999999999, 22.5); +INSERT INTO boston VALUES (212, 0.37578, 0, 10.59, 1, 0.48899999999999999, 5.4039999999999999, 88.599999999999994, 3.665, 4, 277, 18.600000000000001, 395.24000000000001, 23.98, 19.300000000000001); +INSERT INTO boston VALUES (216, 0.19802, 0, 10.59, 0, 0.48899999999999999, 6.1820000000000004, 42.399999999999999, 3.9453999999999998, 4, 277, 18.600000000000001, 393.63, 9.4700000000000006, 25); +INSERT INTO boston VALUES (220, 0.11425, 0, 13.890000000000001, 1, 0.55000000000000004, 6.3730000000000002, 92.400000000000006, 3.3633000000000002, 5, 276, 16.399999999999999, 393.74000000000001, 10.5, 23); +INSERT INTO boston VALUES (224, 0.61470000000000002, 0, 6.2000000000000002, 0, 0.50700000000000001, 6.6180000000000003, 80.799999999999997, 3.2721, 8, 307, 17.399999999999999, 396.89999999999998, 7.5999999999999996, 30.100000000000001); +INSERT INTO boston VALUES (228, 0.41238000000000002, 0, 6.2000000000000002, 0, 0.504, 7.1630000000000003, 79.900000000000006, 3.2157, 8, 307, 17.399999999999999, 372.07999999999998, 6.3600000000000003, 31.600000000000001); +INSERT INTO boston VALUES (232, 0.46295999999999998, 0, 6.2000000000000002, 0, 0.504, 7.4119999999999999, 76.900000000000006, 3.6715, 8, 307, 17.399999999999999, 376.13999999999999, 5.25, 31.699999999999999); +INSERT INTO boston VALUES (233, 0.57528999999999997, 0, 6.2000000000000002, 0, 0.50700000000000001, 8.3369999999999997, 73.299999999999997, 3.8384, 8, 307, 17.399999999999999, 385.91000000000003, 2.4700000000000002, 41.700000000000003); +INSERT INTO boston VALUES (237, 0.52058000000000004, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.6310000000000002, 76.5, 4.1479999999999997, 8, 307, 17.399999999999999, 388.44999999999999, 9.5399999999999991, 25.100000000000001); +INSERT INTO boston VALUES (241, 0.11329, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.8970000000000002, 54.299999999999997, 6.3361000000000001, 6, 300, 16.600000000000001, 391.25, 11.380000000000001, 22); +INSERT INTO boston VALUES (245, 0.20608000000000001, 22, 5.8600000000000003, 0, 0.43099999999999999, 5.593, 76.5, 7.9549000000000003, 7, 330, 19.100000000000001, 372.49000000000001, 12.5, 17.600000000000001); +INSERT INTO boston VALUES (249, 0.16439000000000001, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.4329999999999998, 49.100000000000001, 7.8265000000000002, 7, 330, 19.100000000000001, 374.70999999999998, 9.5199999999999996, 24.5); +INSERT INTO boston VALUES (253, 0.082210000000000005, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.9569999999999999, 6.7999999999999998, 8.9067000000000007, 7, 330, 19.100000000000001, 386.08999999999997, 3.5299999999999998, 29.600000000000001); +INSERT INTO boston VALUES (258, 0.61153999999999997, 20, 3.9700000000000002, 0, 0.64700000000000002, 8.7040000000000006, 86.900000000000006, 1.8009999999999999, 5, 264, 13, 389.69999999999999, 5.1200000000000001, 50); +INSERT INTO boston VALUES (262, 0.53412000000000004, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.5199999999999996, 89.400000000000006, 2.1398000000000001, 5, 264, 13, 388.37, 7.2599999999999998, 43.100000000000001); +INSERT INTO boston VALUES (266, 0.76161999999999996, 20, 3.9700000000000002, 0, 0.64700000000000002, 5.5599999999999996, 62.799999999999997, 1.9864999999999999, 5, 264, 13, 392.39999999999998, 10.449999999999999, 22.800000000000001); +INSERT INTO boston VALUES (270, 0.090649999999999994, 20, 6.96, 1, 0.46400000000000002, 5.9199999999999999, 61.5, 3.9175, 3, 223, 18.600000000000001, 391.33999999999997, 13.65, 20.699999999999999); +INSERT INTO boston VALUES (274, 0.22187999999999999, 20, 6.96, 1, 0.46400000000000002, 7.6909999999999998, 51.799999999999997, 4.3665000000000003, 3, 223, 18.600000000000001, 390.76999999999998, 6.5800000000000001, 35.200000000000003); +INSERT INTO boston VALUES (278, 0.061269999999999998, 40, 6.4100000000000001, 1, 0.44700000000000001, 6.8259999999999996, 27.600000000000001, 4.8628, 4, 254, 17.600000000000001, 393.44999999999999, 4.1600000000000001, 33.100000000000001); +INSERT INTO boston VALUES (282, 0.03705, 20, 3.3300000000000001, 0, 0.44290000000000002, 6.968, 37.200000000000003, 5.2446999999999999, 5, 216, 14.9, 392.23000000000002, 4.5899999999999999, 35.399999999999999); +INSERT INTO boston VALUES (286, 0.010959999999999999, 55, 2.25, 0, 0.38900000000000001, 6.4530000000000003, 31.899999999999999, 7.3072999999999997, 1, 300, 15.300000000000001, 394.72000000000003, 8.2300000000000004, 22); +INSERT INTO boston VALUES (290, 0.042970000000000001, 52.5, 5.3200000000000003, 0, 0.40500000000000003, 6.5650000000000004, 22.899999999999999, 7.3171999999999997, 6, 293, 16.600000000000001, 371.72000000000003, 9.5099999999999998, 24.800000000000001); +INSERT INTO boston VALUES (294, 0.082650000000000001, 0, 13.92, 0, 0.437, 6.1269999999999998, 18.399999999999999, 5.5026999999999999, 4, 289, 16, 396.89999999999998, 8.5800000000000001, 23.899999999999999); +INSERT INTO boston VALUES (298, 0.14102999999999999, 0, 13.92, 0, 0.437, 5.79, 58, 6.3200000000000003, 4, 289, 16, 396.89999999999998, 15.84, 20.300000000000001); +INSERT INTO boston VALUES (302, 0.035369999999999999, 34, 6.0899999999999999, 0, 0.433, 6.5899999999999999, 40.399999999999999, 5.4916999999999998, 7, 329, 16.100000000000001, 395.75, 9.5, 22); +INSERT INTO boston VALUES (306, 0.054789999999999998, 33, 2.1800000000000002, 0, 0.47199999999999998, 6.6159999999999997, 58.100000000000001, 3.3700000000000001, 7, 222, 18.399999999999999, 393.36000000000001, 8.9299999999999997, 28.399999999999999); +INSERT INTO boston VALUES (310, 0.34939999999999999, 0, 9.9000000000000004, 0, 0.54400000000000004, 5.9720000000000004, 76.700000000000003, 3.1025, 4, 304, 18.399999999999999, 396.24000000000001, 9.9700000000000006, 20.300000000000001); +INSERT INTO boston VALUES (314, 0.26938000000000001, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.266, 82.799999999999997, 3.2627999999999999, 4, 304, 18.399999999999999, 393.38999999999999, 7.9000000000000004, 21.600000000000001); +INSERT INTO boston VALUES (318, 0.24521999999999999, 0, 9.9000000000000004, 0, 0.54400000000000004, 5.782, 71.700000000000003, 4.0316999999999998, 4, 304, 18.399999999999999, 396.89999999999998, 15.94, 19.800000000000001); +INSERT INTO boston VALUES (322, 0.18159, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.3760000000000003, 54.299999999999997, 4.5404, 5, 287, 19.600000000000001, 396.89999999999998, 6.8700000000000001, 23.100000000000001); +INSERT INTO boston VALUES (326, 0.19186, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.431, 14.699999999999999, 5.4158999999999997, 5, 287, 19.600000000000001, 393.68000000000001, 5.0800000000000001, 24.600000000000001); +INSERT INTO boston VALUES (330, 0.067239999999999994, 0, 3.2400000000000002, 0, 0.46000000000000002, 6.3330000000000002, 17.199999999999999, 5.2145999999999999, 4, 430, 16.899999999999999, 375.20999999999998, 7.3399999999999999, 22.600000000000001); +INSERT INTO boston VALUES (334, 0.05083, 0, 5.1900000000000004, 0, 0.51500000000000001, 6.3159999999999998, 38.100000000000001, 6.4584000000000001, 5, 224, 20.199999999999999, 389.70999999999998, 5.6799999999999997, 22.199999999999999); +INSERT INTO boston VALUES (338, 0.03041, 0, 5.1900000000000004, 0, 0.51500000000000001, 5.8949999999999996, 59.600000000000001, 5.6150000000000002, 5, 224, 20.199999999999999, 394.81, 10.56, 18.5); +INSERT INTO boston VALUES (342, 0.013010000000000001, 35, 1.52, 0, 0.442, 7.2409999999999997, 49.299999999999997, 7.0378999999999996, 1, 284, 15.5, 394.74000000000001, 5.4900000000000002, 32.700000000000003); +INSERT INTO boston VALUES (346, 0.031130000000000001, 0, 4.3899999999999997, 0, 0.442, 6.0140000000000002, 48.5, 8.0136000000000003, 3, 352, 18.800000000000001, 385.63999999999999, 10.529999999999999, 17.5); +INSERT INTO boston VALUES (350, 0.028989999999999998, 40, 1.25, 0, 0.42899999999999999, 6.9390000000000001, 34.5, 8.7920999999999996, 1, 335, 19.699999999999999, 389.85000000000002, 5.8899999999999997, 26.600000000000001); +INSERT INTO boston VALUES (354, 0.017090000000000001, 90, 2.02, 0, 0.40999999999999998, 6.7279999999999998, 36.100000000000001, 12.1265, 5, 187, 17, 384.45999999999998, 4.5, 30.100000000000001); +INSERT INTO boston VALUES (358, 3.8496999999999999, 0, 18.100000000000001, 1, 0.77000000000000002, 6.3949999999999996, 91, 2.5051999999999999, 24, 666, 20.199999999999999, 391.33999999999997, 13.27, 21.699999999999999); +INSERT INTO boston VALUES (362, 3.83684, 0, 18.100000000000001, 0, 0.77000000000000002, 6.2510000000000003, 91.099999999999994, 2.2955000000000001, 24, 666, 20.199999999999999, 350.64999999999998, 14.19, 19.899999999999999); +INSERT INTO boston VALUES (366, 4.5558699999999996, 0, 18.100000000000001, 0, 0.71799999999999997, 3.5609999999999999, 87.900000000000006, 1.6132, 24, 666, 20.199999999999999, 354.69999999999999, 7.1200000000000001, 27.5); +INSERT INTO boston VALUES (370, 5.6699799999999998, 0, 18.100000000000001, 1, 0.63100000000000001, 6.6829999999999998, 96.799999999999997, 1.3567, 24, 666, 20.199999999999999, 375.32999999999998, 3.73, 50); +INSERT INTO boston VALUES (374, 11.1081, 0, 18.100000000000001, 0, 0.66800000000000004, 4.9059999999999997, 100, 1.1741999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 34.770000000000003, 13.800000000000001); +INSERT INTO boston VALUES (378, 9.8234899999999996, 0, 18.100000000000001, 0, 0.67100000000000004, 6.7939999999999996, 98.799999999999997, 1.3580000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 21.239999999999998, 13.300000000000001); +INSERT INTO boston VALUES (382, 15.8744, 0, 18.100000000000001, 0, 0.67100000000000004, 6.5449999999999999, 99.099999999999994, 1.5192000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 21.079999999999998, 10.9); +INSERT INTO boston VALUES (386, 16.811800000000002, 0, 18.100000000000001, 0, 0.69999999999999996, 5.2770000000000001, 98.099999999999994, 1.4260999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 30.809999999999999, 7.2000000000000002); +INSERT INTO boston VALUES (390, 8.1517400000000002, 0, 18.100000000000001, 0, 0.69999999999999996, 5.3899999999999997, 98.900000000000006, 1.7281, 24, 666, 20.199999999999999, 396.89999999999998, 20.850000000000001, 11.5); +INSERT INTO boston VALUES (394, 8.6447599999999998, 0, 18.100000000000001, 0, 0.69299999999999995, 6.1929999999999996, 92.599999999999994, 1.7911999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 15.17, 13.800000000000001); +INSERT INTO boston VALUES (398, 7.6720199999999998, 0, 18.100000000000001, 0, 0.69299999999999995, 5.7469999999999999, 98.900000000000006, 1.6334, 24, 666, 20.199999999999999, 393.10000000000002, 19.920000000000002, 8.5); +INSERT INTO boston VALUES (402, 14.2362, 0, 18.100000000000001, 0, 0.69299999999999995, 6.343, 100, 1.5741000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 20.32, 7.2000000000000002); +INSERT INTO boston VALUES (406, 67.9208, 0, 18.100000000000001, 0, 0.69299999999999995, 5.6829999999999998, 100, 1.4254, 24, 666, 20.199999999999999, 384.97000000000003, 22.98, 5); +INSERT INTO boston VALUES (410, 14.4383, 0, 18.100000000000001, 0, 0.59699999999999998, 6.8520000000000003, 100, 1.4655, 24, 666, 20.199999999999999, 179.36000000000001, 19.780000000000001, 27.5); +INSERT INTO boston VALUES (414, 28.655799999999999, 0, 18.100000000000001, 0, 0.59699999999999998, 5.1550000000000002, 100, 1.5893999999999999, 24, 666, 20.199999999999999, 210.97, 20.079999999999998, 16.300000000000001); +INSERT INTO boston VALUES (418, 25.9406, 0, 18.100000000000001, 0, 0.67900000000000005, 5.3040000000000003, 89.099999999999994, 1.6475, 24, 666, 20.199999999999999, 127.36, 26.640000000000001, 10.4); +INSERT INTO boston VALUES (422, 7.0225900000000001, 0, 18.100000000000001, 0, 0.71799999999999997, 6.0060000000000002, 95.299999999999997, 1.8746, 24, 666, 20.199999999999999, 319.98000000000002, 15.699999999999999, 14.199999999999999); +INSERT INTO boston VALUES (426, 15.860300000000001, 0, 18.100000000000001, 0, 0.67900000000000005, 5.8959999999999999, 95.400000000000006, 1.9096, 24, 666, 20.199999999999999, 7.6799999999999997, 24.390000000000001, 8.3000000000000007); +INSERT INTO boston VALUES (430, 9.3388899999999992, 0, 18.100000000000001, 0, 0.67900000000000005, 6.3799999999999999, 95.599999999999994, 1.9681999999999999, 24, 666, 20.199999999999999, 60.719999999999999, 24.079999999999998, 9.5); +INSERT INTO boston VALUES (434, 5.5810700000000004, 0, 18.100000000000001, 0, 0.71299999999999997, 6.4359999999999999, 87.900000000000006, 2.3157999999999999, 24, 666, 20.199999999999999, 100.19, 16.219999999999999, 14.300000000000001); +INSERT INTO boston VALUES (438, 15.177199999999999, 0, 18.100000000000001, 0, 0.73999999999999999, 6.1520000000000001, 100, 1.9141999999999999, 24, 666, 20.199999999999999, 9.3200000000000003, 26.449999999999999, 8.6999999999999993); +INSERT INTO boston VALUES (442, 9.7241800000000005, 0, 18.100000000000001, 0, 0.73999999999999999, 6.4059999999999997, 97.200000000000003, 2.0651000000000002, 24, 666, 20.199999999999999, 385.95999999999998, 19.52, 17.100000000000001); +INSERT INTO boston VALUES (446, 10.671799999999999, 0, 18.100000000000001, 0, 0.73999999999999999, 6.4589999999999996, 94.799999999999997, 1.9879, 24, 666, 20.199999999999999, 43.060000000000002, 23.98, 11.800000000000001); +INSERT INTO boston VALUES (450, 7.5260100000000003, 0, 18.100000000000001, 0, 0.71299999999999997, 6.4169999999999998, 98.299999999999997, 2.1850000000000001, 24, 666, 20.199999999999999, 304.20999999999998, 19.309999999999999, 13); +INSERT INTO boston VALUES (454, 8.2480899999999995, 0, 18.100000000000001, 0, 0.71299999999999997, 7.3929999999999998, 99.299999999999997, 2.4527000000000001, 24, 666, 20.199999999999999, 375.87, 16.739999999999998, 17.800000000000001); +INSERT INTO boston VALUES (458, 8.2005800000000004, 0, 18.100000000000001, 0, 0.71299999999999997, 5.9359999999999999, 80.299999999999997, 2.7791999999999999, 24, 666, 20.199999999999999, 3.5, 16.940000000000001, 13.5); +INSERT INTO boston VALUES (462, 3.6931099999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.3760000000000003, 88.400000000000006, 2.5670999999999999, 24, 666, 20.199999999999999, 391.43000000000001, 14.65, 17.699999999999999); +INSERT INTO boston VALUES (465, 7.8393199999999998, 0, 18.100000000000001, 0, 0.65500000000000003, 6.2089999999999996, 65.400000000000006, 2.9634, 24, 666, 20.199999999999999, 396.89999999999998, 13.220000000000001, 21.399999999999999); +INSERT INTO boston VALUES (469, 15.575699999999999, 0, 18.100000000000001, 0, 0.57999999999999996, 5.9260000000000002, 71, 2.9083999999999999, 24, 666, 20.199999999999999, 368.74000000000001, 18.129999999999999, 19.100000000000001); +INSERT INTO boston VALUES (473, 3.5686800000000001, 0, 18.100000000000001, 0, 0.57999999999999996, 6.4370000000000003, 75, 2.8965000000000001, 24, 666, 20.199999999999999, 393.37, 14.359999999999999, 23.199999999999999); +INSERT INTO boston VALUES (477, 4.87141, 0, 18.100000000000001, 0, 0.61399999999999999, 6.484, 93.599999999999994, 2.3052999999999999, 24, 666, 20.199999999999999, 396.20999999999998, 18.68, 16.699999999999999); +INSERT INTO boston VALUES (481, 5.8240100000000004, 0, 18.100000000000001, 0, 0.53200000000000003, 6.242, 64.700000000000003, 3.4241999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 10.74, 23); +INSERT INTO boston VALUES (485, 2.3785699999999999, 0, 18.100000000000001, 0, 0.58299999999999996, 5.8710000000000004, 41.899999999999999, 3.7240000000000002, 24, 666, 20.199999999999999, 370.73000000000002, 13.34, 20.600000000000001); +INSERT INTO boston VALUES (489, 0.15085999999999999, 0, 27.739999999999998, 0, 0.60899999999999999, 5.4539999999999997, 92.700000000000003, 1.8209, 4, 711, 20.100000000000001, 395.08999999999997, 18.059999999999999, 15.199999999999999); +INSERT INTO boston VALUES (493, 0.11132, 0, 27.739999999999998, 0, 0.60899999999999999, 5.9829999999999997, 83.5, 2.1099000000000001, 4, 711, 20.100000000000001, 396.89999999999998, 13.35, 20.100000000000001); +INSERT INTO boston VALUES (497, 0.28960000000000002, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.3899999999999997, 72.900000000000006, 2.7986, 6, 391, 19.199999999999999, 396.89999999999998, 21.140000000000001, 19.699999999999999); +INSERT INTO boston VALUES (501, 0.22438, 0, 9.6899999999999995, 0, 0.58499999999999996, 6.0270000000000001, 79.700000000000003, 2.4982000000000002, 6, 391, 19.199999999999999, 396.89999999999998, 14.33, 16.800000000000001); +INSERT INTO boston VALUES (505, 0.10959000000000001, 0, 11.93, 0, 0.57299999999999995, 6.7939999999999996, 89.299999999999997, 2.3889, 1, 273, 21, 393.44999999999999, 6.4800000000000004, 22); +INSERT INTO boston VALUES (235, 0.44790999999999997, 0, 6.2000000000000002, 1, 0.50700000000000001, 6.726, 66.5, 3.6518999999999999, 8, 307, 17.399999999999999, 360.19999999999999, 8.0500000000000007, 29); +INSERT INTO boston VALUES (239, 0.082439999999999999, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.4809999999999999, 18.5, 6.1898999999999997, 6, 300, 16.600000000000001, 379.41000000000003, 6.3600000000000003, 23.699999999999999); +INSERT INTO boston VALUES (243, 0.10290000000000001, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.3579999999999997, 52.899999999999999, 7.0354999999999999, 6, 300, 16.600000000000001, 372.75, 11.220000000000001, 22.199999999999999); +INSERT INTO boston VALUES (247, 0.33983000000000002, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.1079999999999997, 34.899999999999999, 8.0555000000000003, 7, 330, 19.100000000000001, 390.18000000000001, 9.1600000000000001, 24.300000000000001); +INSERT INTO boston VALUES (251, 0.14030000000000001, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.4870000000000001, 13, 7.3967000000000001, 7, 330, 19.100000000000001, 396.27999999999997, 5.9000000000000004, 24.399999999999999); +INSERT INTO boston VALUES (255, 0.048189999999999997, 80, 3.6400000000000001, 0, 0.39200000000000002, 6.1079999999999997, 32, 9.2202999999999999, 1, 315, 16.399999999999999, 392.88999999999999, 6.5700000000000003, 21.899999999999999); +INSERT INTO boston VALUES (256, 0.035479999999999998, 80, 3.6400000000000001, 0, 0.39200000000000002, 5.8760000000000003, 19.100000000000001, 9.2202999999999999, 1, 315, 16.399999999999999, 395.18000000000001, 9.25, 20.899999999999999); +INSERT INTO boston VALUES (260, 0.65664999999999996, 20, 3.9700000000000002, 0, 0.64700000000000002, 6.8419999999999996, 100, 2.0106999999999999, 5, 264, 13, 391.93000000000001, 6.9000000000000004, 30.100000000000001); +INSERT INTO boston VALUES (264, 0.82525999999999999, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.327, 94.5, 2.0788000000000002, 5, 264, 13, 393.42000000000002, 11.25, 31); +INSERT INTO boston VALUES (268, 0.57833999999999997, 20, 3.9700000000000002, 0, 0.57499999999999996, 8.2970000000000006, 67, 2.4216000000000002, 5, 264, 13, 384.54000000000002, 7.4400000000000004, 50); +INSERT INTO boston VALUES (272, 0.16211, 20, 6.96, 0, 0.46400000000000002, 6.2400000000000002, 16.300000000000001, 4.4290000000000003, 3, 223, 18.600000000000001, 396.89999999999998, 6.5899999999999999, 25.199999999999999); +INSERT INTO boston VALUES (276, 0.09604, 40, 6.4100000000000001, 0, 0.44700000000000001, 6.8540000000000001, 42.799999999999997, 4.2672999999999996, 4, 254, 17.600000000000001, 396.89999999999998, 2.98, 32); +INSERT INTO boston VALUES (280, 0.21038000000000001, 20, 3.3300000000000001, 0, 0.44290000000000002, 6.8120000000000003, 32.200000000000003, 4.1006999999999998, 5, 216, 14.9, 396.89999999999998, 4.8499999999999996, 35.100000000000001); +INSERT INTO boston VALUES (284, 0.015010000000000001, 90, 1.21, 1, 0.40100000000000002, 7.923, 24.800000000000001, 5.8849999999999998, 1, 198, 13.6, 395.51999999999998, 3.1600000000000001, 50); +INSERT INTO boston VALUES (288, 0.038710000000000001, 52.5, 5.3200000000000003, 0, 0.40500000000000003, 6.2089999999999996, 31.300000000000001, 7.3171999999999997, 6, 293, 16.600000000000001, 396.89999999999998, 7.1399999999999997, 23.199999999999999); +INSERT INTO boston VALUES (292, 0.07886, 80, 4.9500000000000002, 0, 0.41099999999999998, 7.1479999999999997, 27.699999999999999, 5.1166999999999998, 4, 245, 19.199999999999999, 396.89999999999998, 3.5600000000000001, 37.299999999999997); +INSERT INTO boston VALUES (296, 0.12931999999999999, 0, 13.92, 0, 0.437, 6.6779999999999999, 31.100000000000001, 5.9603999999999999, 4, 289, 16, 396.89999999999998, 6.2699999999999996, 28.600000000000001); +INSERT INTO boston VALUES (300, 0.05561, 70, 2.2400000000000002, 0, 0.40000000000000002, 7.0410000000000004, 10, 7.8277999999999999, 5, 358, 14.800000000000001, 371.57999999999998, 4.7400000000000002, 29); +INSERT INTO boston VALUES (304, 0.10000000000000001, 34, 6.0899999999999999, 0, 0.433, 6.9820000000000002, 17.699999999999999, 5.4916999999999998, 7, 329, 16.100000000000001, 390.43000000000001, 4.8600000000000003, 33.100000000000001); +INSERT INTO boston VALUES (308, 0.049320000000000003, 33, 2.1800000000000002, 0, 0.47199999999999998, 6.8490000000000002, 70.299999999999997, 3.1827000000000001, 7, 222, 18.399999999999999, 396.89999999999998, 7.5300000000000002, 28.199999999999999); +INSERT INTO boston VALUES (312, 0.79040999999999995, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.1219999999999999, 52.799999999999997, 2.6402999999999999, 4, 304, 18.399999999999999, 396.89999999999998, 5.9800000000000004, 22.100000000000001); +INSERT INTO boston VALUES (316, 0.25356000000000001, 0, 9.9000000000000004, 0, 0.54400000000000004, 5.7050000000000001, 77.700000000000003, 3.9449999999999998, 4, 304, 18.399999999999999, 396.42000000000002, 11.5, 16.199999999999999); +INSERT INTO boston VALUES (320, 0.47547, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.1130000000000004, 58.799999999999997, 4.0019, 4, 304, 18.399999999999999, 396.23000000000002, 12.73, 21); +INSERT INTO boston VALUES (324, 0.28392000000000001, 0, 7.3799999999999999, 0, 0.49299999999999999, 5.7080000000000002, 74.299999999999997, 4.7210999999999999, 5, 287, 19.600000000000001, 391.13, 11.74, 18.5); +INSERT INTO boston VALUES (328, 0.24102999999999999, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.0830000000000002, 43.700000000000003, 5.4158999999999997, 5, 287, 19.600000000000001, 396.89999999999998, 12.789999999999999, 22.199999999999999); +INSERT INTO boston VALUES (332, 0.050229999999999997, 35, 6.0599999999999996, 0, 0.43790000000000001, 5.7060000000000004, 28.399999999999999, 6.6406999999999998, 1, 304, 16.899999999999999, 394.01999999999998, 12.43, 17.100000000000001); +INSERT INTO boston VALUES (336, 0.039609999999999999, 0, 5.1900000000000004, 0, 0.51500000000000001, 6.0369999999999999, 34.5, 5.9852999999999996, 5, 224, 20.199999999999999, 396.89999999999998, 8.0099999999999998, 21.100000000000001); +INSERT INTO boston VALUES (340, 0.054969999999999998, 0, 5.1900000000000004, 0, 0.51500000000000001, 5.9850000000000003, 45.399999999999999, 4.8121999999999998, 5, 224, 20.199999999999999, 396.89999999999998, 9.7400000000000002, 19); +INSERT INTO boston VALUES (344, 0.025430000000000001, 55, 3.7799999999999998, 0, 0.48399999999999999, 6.6959999999999997, 56.399999999999999, 5.7321, 5, 370, 17.600000000000001, 396.89999999999998, 7.1799999999999997, 23.899999999999999); +INSERT INTO boston VALUES (348, 0.018700000000000001, 85, 4.1500000000000004, 0, 0.42899999999999999, 6.516, 27.699999999999999, 8.5352999999999994, 4, 351, 17.899999999999999, 392.43000000000001, 6.3600000000000003, 23.100000000000001); +INSERT INTO boston VALUES (352, 0.079500000000000001, 60, 1.6899999999999999, 0, 0.41099999999999998, 6.5789999999999997, 35.899999999999999, 10.7103, 4, 411, 18.300000000000001, 370.77999999999997, 5.4900000000000002, 24.100000000000001); +INSERT INTO boston VALUES (356, 0.10659, 80, 1.9099999999999999, 0, 0.41299999999999998, 5.9359999999999999, 19.5, 10.585699999999999, 4, 334, 22, 376.04000000000002, 5.5700000000000003, 20.600000000000001); +INSERT INTO boston VALUES (360, 4.2613099999999999, 0, 18.100000000000001, 0, 0.77000000000000002, 6.1120000000000001, 81.299999999999997, 2.5091000000000001, 24, 666, 20.199999999999999, 390.74000000000001, 12.67, 22.600000000000001); +INSERT INTO boston VALUES (364, 4.2223899999999999, 0, 18.100000000000001, 1, 0.77000000000000002, 5.8029999999999999, 89, 1.9047000000000001, 24, 666, 20.199999999999999, 353.04000000000002, 14.640000000000001, 16.800000000000001); +INSERT INTO boston VALUES (368, 13.5222, 0, 18.100000000000001, 0, 0.63100000000000001, 3.863, 100, 1.5105999999999999, 24, 666, 20.199999999999999, 131.41999999999999, 13.33, 23.100000000000001); +INSERT INTO boston VALUES (372, 9.2323000000000004, 0, 18.100000000000001, 0, 0.63100000000000001, 6.2160000000000002, 100, 1.1691, 24, 666, 20.199999999999999, 366.14999999999998, 9.5299999999999994, 50); +INSERT INTO boston VALUES (376, 19.609100000000002, 0, 18.100000000000001, 0, 0.67100000000000004, 7.3129999999999997, 97.900000000000006, 1.3163, 24, 666, 20.199999999999999, 396.89999999999998, 13.44, 15); +INSERT INTO boston VALUES (380, 17.866700000000002, 0, 18.100000000000001, 0, 0.67100000000000004, 6.2229999999999999, 100, 1.3861000000000001, 24, 666, 20.199999999999999, 393.74000000000001, 21.780000000000001, 10.199999999999999); +INSERT INTO boston VALUES (384, 7.9924799999999996, 0, 18.100000000000001, 0, 0.69999999999999996, 5.5199999999999996, 100, 1.5330999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 24.559999999999999, 12.300000000000001); +INSERT INTO boston VALUES (388, 22.597100000000001, 0, 18.100000000000001, 0, 0.69999999999999996, 5, 89.5, 1.5184, 24, 666, 20.199999999999999, 396.89999999999998, 31.989999999999998, 7.4000000000000004); +INSERT INTO boston VALUES (392, 5.29305, 0, 18.100000000000001, 0, 0.69999999999999996, 6.0510000000000002, 82.5, 2.1678000000000002, 24, 666, 20.199999999999999, 378.38, 18.760000000000002, 23.199999999999999); +INSERT INTO boston VALUES (396, 8.7167499999999993, 0, 18.100000000000001, 0, 0.69299999999999995, 6.4710000000000001, 98.799999999999997, 1.7257, 24, 666, 20.199999999999999, 391.98000000000002, 17.120000000000001, 13.1); +INSERT INTO boston VALUES (400, 9.9165500000000009, 0, 18.100000000000001, 0, 0.69299999999999995, 5.8520000000000003, 77.799999999999997, 1.5004, 24, 666, 20.199999999999999, 338.16000000000003, 29.969999999999999, 6.2999999999999998); +INSERT INTO boston VALUES (404, 24.8017, 0, 18.100000000000001, 0, 0.69299999999999995, 5.3490000000000002, 96, 1.7028000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 19.77, 8.3000000000000007); +INSERT INTO boston VALUES (408, 11.9511, 0, 18.100000000000001, 0, 0.65900000000000003, 5.6079999999999997, 100, 1.2851999999999999, 24, 666, 20.199999999999999, 332.08999999999997, 12.130000000000001, 27.899999999999999); +INSERT INTO boston VALUES (412, 14.050700000000001, 0, 18.100000000000001, 0, 0.59699999999999998, 6.657, 100, 1.5275000000000001, 24, 666, 20.199999999999999, 35.049999999999997, 21.219999999999999, 17.199999999999999); +INSERT INTO boston VALUES (416, 18.084599999999998, 0, 18.100000000000001, 0, 0.67900000000000005, 6.4340000000000002, 100, 1.8347, 24, 666, 20.199999999999999, 27.25, 29.050000000000001, 7.2000000000000002); +INSERT INTO boston VALUES (420, 11.8123, 0, 18.100000000000001, 0, 0.71799999999999997, 6.8239999999999998, 76.5, 1.794, 24, 666, 20.199999999999999, 48.450000000000003, 22.739999999999998, 8.4000000000000004); +INSERT INTO boston VALUES (424, 7.0504199999999999, 0, 18.100000000000001, 0, 0.61399999999999999, 6.1029999999999998, 85.099999999999994, 2.0217999999999998, 24, 666, 20.199999999999999, 2.52, 23.289999999999999, 13.4); +INSERT INTO boston VALUES (428, 37.661900000000003, 0, 18.100000000000001, 0, 0.67900000000000005, 6.202, 78.700000000000003, 1.8629, 24, 666, 20.199999999999999, 18.82, 14.52, 10.9); +INSERT INTO boston VALUES (432, 10.0623, 0, 18.100000000000001, 0, 0.58399999999999996, 6.8330000000000002, 94.299999999999997, 2.0882000000000001, 24, 666, 20.199999999999999, 81.329999999999998, 19.690000000000001, 14.1); +INSERT INTO boston VALUES (436, 11.160399999999999, 0, 18.100000000000001, 0, 0.73999999999999999, 6.6289999999999996, 94.599999999999994, 2.1246999999999998, 24, 666, 20.199999999999999, 109.84999999999999, 23.27, 13.4); +INSERT INTO boston VALUES (440, 9.3906299999999998, 0, 18.100000000000001, 0, 0.73999999999999999, 5.6269999999999998, 93.900000000000006, 1.8171999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 22.879999999999999, 12.800000000000001); +INSERT INTO boston VALUES (444, 9.9665400000000002, 0, 18.100000000000001, 0, 0.73999999999999999, 6.4850000000000003, 100, 1.9783999999999999, 24, 666, 20.199999999999999, 386.73000000000002, 18.850000000000001, 15.4); +INSERT INTO boston VALUES (448, 9.9248499999999993, 0, 18.100000000000001, 0, 0.73999999999999999, 6.2510000000000003, 96.599999999999994, 2.198, 24, 666, 20.199999999999999, 388.51999999999998, 16.440000000000001, 12.6); +INSERT INTO boston VALUES (452, 5.4411399999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.6550000000000002, 98.200000000000003, 2.3552, 24, 666, 20.199999999999999, 355.29000000000002, 17.73, 15.199999999999999); +INSERT INTO boston VALUES (456, 4.75237, 0, 18.100000000000001, 0, 0.71299999999999997, 6.5250000000000004, 86.5, 2.4358, 24, 666, 20.199999999999999, 50.920000000000002, 18.129999999999999, 14.1); +INSERT INTO boston VALUES (460, 6.8011699999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.0810000000000004, 84.400000000000006, 2.7174999999999998, 24, 666, 20.199999999999999, 396.89999999999998, 14.699999999999999, 20); +INSERT INTO boston VALUES (236, 0.33045000000000002, 0, 6.2000000000000002, 0, 0.50700000000000001, 6.0860000000000003, 61.5, 3.6518999999999999, 8, 307, 17.399999999999999, 376.75, 10.880000000000001, 24); +INSERT INTO boston VALUES (240, 0.092520000000000005, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.6059999999999999, 42.200000000000003, 6.1898999999999997, 6, 300, 16.600000000000001, 383.77999999999997, 7.3700000000000001, 23.300000000000001); +INSERT INTO boston VALUES (244, 0.12756999999999999, 30, 4.9299999999999997, 0, 0.42799999999999999, 6.3929999999999998, 7.7999999999999998, 7.0354999999999999, 6, 300, 16.600000000000001, 374.70999999999998, 5.1900000000000004, 23.699999999999999); +INSERT INTO boston VALUES (248, 0.19656999999999999, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.226, 79.200000000000003, 8.0555000000000003, 7, 330, 19.100000000000001, 376.13999999999999, 10.15, 20.5); +INSERT INTO boston VALUES (252, 0.21409, 22, 5.8600000000000003, 0, 0.43099999999999999, 6.4379999999999997, 8.9000000000000004, 7.3967000000000001, 7, 330, 19.100000000000001, 377.06999999999999, 3.5899999999999999, 24.800000000000001); +INSERT INTO boston VALUES (259, 0.66351000000000004, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.3330000000000002, 100, 1.8946000000000001, 5, 264, 13, 383.29000000000002, 7.79, 36); +INSERT INTO boston VALUES (263, 0.52014000000000005, 20, 3.9700000000000002, 0, 0.64700000000000002, 8.3979999999999997, 91.5, 2.2885, 5, 264, 13, 386.86000000000001, 5.9100000000000001, 48.799999999999997); +INSERT INTO boston VALUES (267, 0.78569999999999995, 20, 3.9700000000000002, 0, 0.64700000000000002, 7.0140000000000002, 84.599999999999994, 2.1328999999999998, 5, 264, 13, 384.06999999999999, 14.789999999999999, 30.699999999999999); +INSERT INTO boston VALUES (271, 0.29915999999999998, 20, 6.96, 0, 0.46400000000000002, 5.8559999999999999, 42.100000000000001, 4.4290000000000003, 3, 223, 18.600000000000001, 388.64999999999998, 13, 21.100000000000001); +INSERT INTO boston VALUES (275, 0.056439999999999997, 40, 6.4100000000000001, 1, 0.44700000000000001, 6.758, 32.899999999999999, 4.0776000000000003, 4, 254, 17.600000000000001, 396.89999999999998, 3.5299999999999998, 32.399999999999999); +INSERT INTO boston VALUES (279, 0.079780000000000004, 40, 6.4100000000000001, 0, 0.44700000000000001, 6.4820000000000002, 32.100000000000001, 4.1402999999999999, 4, 254, 17.600000000000001, 396.89999999999998, 7.1900000000000004, 29.100000000000001); +INSERT INTO boston VALUES (283, 0.061289999999999997, 20, 3.3300000000000001, 1, 0.44290000000000002, 7.6449999999999996, 49.700000000000003, 5.2119, 5, 216, 14.9, 377.06999999999999, 3.0099999999999998, 46); +INSERT INTO boston VALUES (287, 0.019650000000000001, 80, 1.76, 0, 0.38500000000000001, 6.2300000000000004, 31.5, 9.0891999999999999, 1, 241, 18.199999999999999, 341.60000000000002, 12.93, 20.100000000000001); +INSERT INTO boston VALUES (291, 0.035020000000000003, 80, 4.9500000000000002, 0, 0.41099999999999998, 6.8609999999999998, 27.899999999999999, 5.1166999999999998, 4, 245, 19.199999999999999, 396.89999999999998, 3.3300000000000001, 28.5); +INSERT INTO boston VALUES (295, 0.081989999999999993, 0, 13.92, 0, 0.437, 6.0090000000000003, 42.299999999999997, 5.5026999999999999, 4, 289, 16, 396.89999999999998, 10.4, 21.699999999999999); +INSERT INTO boston VALUES (299, 0.064659999999999995, 70, 2.2400000000000002, 0, 0.40000000000000002, 6.3449999999999998, 20.100000000000001, 7.8277999999999999, 5, 358, 14.800000000000001, 368.24000000000001, 4.9699999999999998, 22.5); +INSERT INTO boston VALUES (303, 0.092660000000000006, 34, 6.0899999999999999, 0, 0.433, 6.4950000000000001, 18.399999999999999, 5.4916999999999998, 7, 329, 16.100000000000001, 383.61000000000001, 8.6699999999999999, 26.399999999999999); +INSERT INTO boston VALUES (307, 0.075029999999999999, 33, 2.1800000000000002, 0, 0.47199999999999998, 7.4199999999999999, 71.900000000000006, 3.0992000000000002, 7, 222, 18.399999999999999, 396.89999999999998, 6.4699999999999998, 33.399999999999999); +INSERT INTO boston VALUES (311, 2.6354799999999998, 0, 9.9000000000000004, 0, 0.54400000000000004, 4.9729999999999999, 37.799999999999997, 2.5194000000000001, 4, 304, 18.399999999999999, 350.44999999999999, 12.640000000000001, 16.100000000000001); +INSERT INTO boston VALUES (315, 0.36919999999999997, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.5670000000000002, 87.299999999999997, 3.6023000000000001, 4, 304, 18.399999999999999, 395.69, 9.2799999999999994, 23.800000000000001); +INSERT INTO boston VALUES (319, 0.40201999999999999, 0, 9.9000000000000004, 0, 0.54400000000000004, 6.3819999999999997, 67.200000000000003, 3.5325000000000002, 4, 304, 18.399999999999999, 395.20999999999998, 10.359999999999999, 23.100000000000001); +INSERT INTO boston VALUES (323, 0.35114000000000001, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.0410000000000004, 49.899999999999999, 4.7210999999999999, 5, 287, 19.600000000000001, 396.89999999999998, 7.7000000000000002, 20.399999999999999); +INSERT INTO boston VALUES (327, 0.30347000000000002, 0, 7.3799999999999999, 0, 0.49299999999999999, 6.3120000000000003, 28.899999999999999, 5.4158999999999997, 5, 287, 19.600000000000001, 396.89999999999998, 6.1500000000000004, 23); +INSERT INTO boston VALUES (331, 0.045440000000000001, 0, 3.2400000000000002, 0, 0.46000000000000002, 6.1440000000000001, 32.200000000000003, 5.8735999999999997, 4, 430, 16.899999999999999, 368.56999999999999, 9.0899999999999999, 19.800000000000001); +INSERT INTO boston VALUES (335, 0.037379999999999997, 0, 5.1900000000000004, 0, 0.51500000000000001, 6.3099999999999996, 38.5, 6.4584000000000001, 5, 224, 20.199999999999999, 389.39999999999998, 6.75, 20.699999999999999); +INSERT INTO boston VALUES (339, 0.033059999999999999, 0, 5.1900000000000004, 0, 0.51500000000000001, 6.0590000000000002, 37.299999999999997, 4.8121999999999998, 5, 224, 20.199999999999999, 396.13999999999999, 8.5099999999999998, 20.600000000000001); +INSERT INTO boston VALUES (343, 0.024979999999999999, 0, 1.8899999999999999, 0, 0.51800000000000002, 6.54, 59.700000000000003, 6.2668999999999997, 1, 422, 15.9, 389.95999999999998, 8.6500000000000004, 16.5); +INSERT INTO boston VALUES (347, 0.061620000000000001, 0, 4.3899999999999997, 0, 0.442, 5.8979999999999997, 52.299999999999997, 8.0136000000000003, 3, 352, 18.800000000000001, 364.61000000000001, 12.67, 17.199999999999999); +INSERT INTO boston VALUES (351, 0.062109999999999999, 40, 1.25, 0, 0.42899999999999999, 6.4900000000000002, 44.399999999999999, 8.7920999999999996, 1, 335, 19.699999999999999, 396.89999999999998, 5.9800000000000004, 22.899999999999999); +INSERT INTO boston VALUES (355, 0.04301, 80, 1.9099999999999999, 0, 0.41299999999999998, 5.6630000000000003, 21.899999999999999, 10.585699999999999, 4, 334, 22, 382.80000000000001, 8.0500000000000007, 18.199999999999999); +INSERT INTO boston VALUES (359, 5.2017699999999998, 0, 18.100000000000001, 1, 0.77000000000000002, 6.1269999999999998, 83.400000000000006, 2.7227000000000001, 24, 666, 20.199999999999999, 395.43000000000001, 11.48, 22.699999999999999); +INSERT INTO boston VALUES (363, 3.67822, 0, 18.100000000000001, 0, 0.77000000000000002, 5.3620000000000001, 96.200000000000003, 2.1036000000000001, 24, 666, 20.199999999999999, 380.79000000000002, 10.19, 20.800000000000001); +INSERT INTO boston VALUES (367, 3.6969500000000002, 0, 18.100000000000001, 0, 0.71799999999999997, 4.9630000000000001, 91.400000000000006, 1.7523, 24, 666, 20.199999999999999, 316.02999999999997, 14, 21.899999999999999); +INSERT INTO boston VALUES (371, 6.5387599999999999, 0, 18.100000000000001, 1, 0.63100000000000001, 7.016, 97.5, 1.2023999999999999, 24, 666, 20.199999999999999, 392.05000000000001, 2.96, 50); +INSERT INTO boston VALUES (375, 18.498200000000001, 0, 18.100000000000001, 0, 0.66800000000000004, 4.1379999999999999, 100, 1.137, 24, 666, 20.199999999999999, 396.89999999999998, 37.969999999999999, 13.800000000000001); +INSERT INTO boston VALUES (379, 23.648199999999999, 0, 18.100000000000001, 0, 0.67100000000000004, 6.3799999999999999, 96.200000000000003, 1.3861000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 23.690000000000001, 13.1); +INSERT INTO boston VALUES (383, 9.1870200000000004, 0, 18.100000000000001, 0, 0.69999999999999996, 5.5359999999999996, 100, 1.5804, 24, 666, 20.199999999999999, 396.89999999999998, 23.600000000000001, 11.300000000000001); +INSERT INTO boston VALUES (387, 24.393799999999999, 0, 18.100000000000001, 0, 0.69999999999999996, 4.6520000000000001, 100, 1.4672000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 28.280000000000001, 10.5); +INSERT INTO boston VALUES (391, 6.9621500000000003, 0, 18.100000000000001, 0, 0.69999999999999996, 5.7130000000000001, 97, 1.9265000000000001, 24, 666, 20.199999999999999, 394.43000000000001, 17.109999999999999, 15.1); +INSERT INTO boston VALUES (395, 13.3598, 0, 18.100000000000001, 0, 0.69299999999999995, 5.8869999999999996, 94.700000000000003, 1.7821, 24, 666, 20.199999999999999, 396.89999999999998, 16.350000000000001, 12.699999999999999); +INSERT INTO boston VALUES (399, 38.351799999999997, 0, 18.100000000000001, 0, 0.69299999999999995, 5.4530000000000003, 100, 1.4896, 24, 666, 20.199999999999999, 396.89999999999998, 30.59, 5); +INSERT INTO boston VALUES (403, 9.5957100000000004, 0, 18.100000000000001, 0, 0.69299999999999995, 6.4039999999999999, 100, 1.639, 24, 666, 20.199999999999999, 376.11000000000001, 20.309999999999999, 12.1); +INSERT INTO boston VALUES (407, 20.716200000000001, 0, 18.100000000000001, 0, 0.65900000000000003, 4.1379999999999999, 100, 1.1780999999999999, 24, 666, 20.199999999999999, 370.22000000000003, 23.34, 11.9); +INSERT INTO boston VALUES (411, 51.135800000000003, 0, 18.100000000000001, 0, 0.59699999999999998, 5.7569999999999997, 100, 1.413, 24, 666, 20.199999999999999, 2.6000000000000001, 10.109999999999999, 15); +INSERT INTO boston VALUES (415, 45.746099999999998, 0, 18.100000000000001, 0, 0.69299999999999995, 4.5190000000000001, 100, 1.6581999999999999, 24, 666, 20.199999999999999, 88.269999999999996, 36.979999999999997, 7); +INSERT INTO boston VALUES (419, 73.534099999999995, 0, 18.100000000000001, 0, 0.67900000000000005, 5.9569999999999999, 100, 1.8026, 24, 666, 20.199999999999999, 16.449999999999999, 20.620000000000001, 8.8000000000000007); +INSERT INTO boston VALUES (423, 12.0482, 0, 18.100000000000001, 0, 0.61399999999999999, 5.6479999999999997, 87.599999999999994, 1.9512, 24, 666, 20.199999999999999, 291.55000000000001, 14.1, 20.800000000000001); +INSERT INTO boston VALUES (427, 12.247199999999999, 0, 18.100000000000001, 0, 0.58399999999999996, 5.8369999999999997, 59.700000000000003, 1.9976, 24, 666, 20.199999999999999, 24.649999999999999, 15.69, 10.199999999999999); +INSERT INTO boston VALUES (431, 8.4921299999999995, 0, 18.100000000000001, 0, 0.58399999999999996, 6.3479999999999999, 86.099999999999994, 2.0527000000000002, 24, 666, 20.199999999999999, 83.450000000000003, 17.640000000000001, 14.5); +INSERT INTO boston VALUES (435, 13.913399999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.2080000000000002, 95, 2.2222, 24, 666, 20.199999999999999, 100.63, 15.17, 11.699999999999999); +INSERT INTO boston VALUES (439, 13.678100000000001, 0, 18.100000000000001, 0, 0.73999999999999999, 5.9349999999999996, 87.900000000000006, 1.8206, 24, 666, 20.199999999999999, 68.950000000000003, 34.020000000000003, 8.4000000000000004); +INSERT INTO boston VALUES (443, 5.6663699999999997, 0, 18.100000000000001, 0, 0.73999999999999999, 6.2190000000000003, 100, 2.0047999999999999, 24, 666, 20.199999999999999, 395.69, 16.59, 18.399999999999999); +INSERT INTO boston VALUES (447, 6.2880700000000003, 0, 18.100000000000001, 0, 0.73999999999999999, 6.3410000000000002, 96.400000000000006, 2.0720000000000001, 24, 666, 20.199999999999999, 318.00999999999999, 17.789999999999999, 14.9); +INSERT INTO boston VALUES (451, 6.7177199999999999, 0, 18.100000000000001, 0, 0.71299999999999997, 6.7489999999999997, 92.599999999999994, 2.3235999999999999, 24, 666, 20.199999999999999, 0.32000000000000001, 17.440000000000001, 13.4); +INSERT INTO boston VALUES (455, 9.5136299999999991, 0, 18.100000000000001, 0, 0.71299999999999997, 6.7279999999999998, 94.099999999999994, 2.4961000000000002, 24, 666, 20.199999999999999, 6.6799999999999997, 18.710000000000001, 14.9); +INSERT INTO boston VALUES (459, 7.75223, 0, 18.100000000000001, 0, 0.71299999999999997, 6.3010000000000002, 83.700000000000003, 2.7831000000000001, 24, 666, 20.199999999999999, 272.20999999999998, 16.23, 14.9); +INSERT INTO boston VALUES (463, 6.6549199999999997, 0, 18.100000000000001, 0, 0.71299999999999997, 6.3170000000000002, 83, 2.7343999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 13.99, 19.5); +INSERT INTO boston VALUES (467, 3.7749799999999998, 0, 18.100000000000001, 0, 0.65500000000000003, 5.952, 84.700000000000003, 2.8715000000000002, 24, 666, 20.199999999999999, 22.010000000000002, 17.149999999999999, 19); +INSERT INTO boston VALUES (466, 3.1636000000000002, 0, 18.100000000000001, 0, 0.65500000000000003, 5.7590000000000003, 48.200000000000003, 3.0665, 24, 666, 20.199999999999999, 334.39999999999998, 14.130000000000001, 19.899999999999999); +INSERT INTO boston VALUES (470, 13.075100000000001, 0, 18.100000000000001, 0, 0.57999999999999996, 5.7130000000000001, 56.700000000000003, 2.8237000000000001, 24, 666, 20.199999999999999, 396.89999999999998, 14.76, 20.100000000000001); +INSERT INTO boston VALUES (474, 4.64689, 0, 18.100000000000001, 0, 0.61399999999999999, 6.9800000000000004, 67.599999999999994, 2.5329000000000002, 24, 666, 20.199999999999999, 374.68000000000001, 11.66, 29.800000000000001); +INSERT INTO boston VALUES (478, 15.023400000000001, 0, 18.100000000000001, 0, 0.61399999999999999, 5.3040000000000003, 97.299999999999997, 2.1006999999999998, 24, 666, 20.199999999999999, 349.48000000000002, 24.91, 12); +INSERT INTO boston VALUES (482, 5.7081799999999996, 0, 18.100000000000001, 0, 0.53200000000000003, 6.75, 74.900000000000006, 3.3317000000000001, 24, 666, 20.199999999999999, 393.06999999999999, 7.7400000000000002, 23.699999999999999); +INSERT INTO boston VALUES (486, 3.67367, 0, 18.100000000000001, 0, 0.58299999999999996, 6.3120000000000003, 51.899999999999999, 3.9916999999999998, 24, 666, 20.199999999999999, 388.62, 10.58, 21.199999999999999); +INSERT INTO boston VALUES (490, 0.18337000000000001, 0, 27.739999999999998, 0, 0.60899999999999999, 5.4139999999999997, 98.299999999999997, 1.7554000000000001, 4, 711, 20.100000000000001, 344.05000000000001, 23.969999999999999, 7); +INSERT INTO boston VALUES (494, 0.17330999999999999, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.7069999999999999, 54, 2.3816999999999999, 6, 391, 19.199999999999999, 396.89999999999998, 12.01, 21.800000000000001); +INSERT INTO boston VALUES (498, 0.26838000000000001, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.7939999999999996, 70.599999999999994, 2.8927, 6, 391, 19.199999999999999, 396.89999999999998, 14.1, 18.300000000000001); +INSERT INTO boston VALUES (502, 0.062630000000000005, 0, 11.93, 0, 0.57299999999999995, 6.593, 69.099999999999994, 2.4786000000000001, 1, 273, 21, 391.99000000000001, 9.6699999999999999, 22.399999999999999); +INSERT INTO boston VALUES (506, 0.047410000000000001, 0, 11.93, 0, 0.57299999999999995, 6.0300000000000002, 80.799999999999997, 2.5049999999999999, 1, 273, 21, 396.89999999999998, 7.8799999999999999, 11.9); +INSERT INTO boston VALUES (464, 5.8211500000000003, 0, 18.100000000000001, 0, 0.71299999999999997, 6.5129999999999999, 89.900000000000006, 2.8016000000000001, 24, 666, 20.199999999999999, 393.81999999999999, 10.289999999999999, 20.199999999999999); +INSERT INTO boston VALUES (468, 4.4222799999999998, 0, 18.100000000000001, 0, 0.58399999999999996, 6.0030000000000001, 94.5, 2.5402999999999998, 24, 666, 20.199999999999999, 331.29000000000002, 21.32, 19.100000000000001); +INSERT INTO boston VALUES (472, 4.0384099999999998, 0, 18.100000000000001, 0, 0.53200000000000003, 6.2290000000000001, 90.700000000000003, 3.0992999999999999, 24, 666, 20.199999999999999, 395.32999999999998, 12.869999999999999, 19.600000000000001); +INSERT INTO boston VALUES (476, 6.3931199999999997, 0, 18.100000000000001, 0, 0.58399999999999996, 6.1619999999999999, 97.400000000000006, 2.206, 24, 666, 20.199999999999999, 302.75999999999999, 24.100000000000001, 13.300000000000001); +INSERT INTO boston VALUES (480, 14.3337, 0, 18.100000000000001, 0, 0.61399999999999999, 6.2290000000000001, 88, 1.9512, 24, 666, 20.199999999999999, 383.31999999999999, 13.109999999999999, 21.399999999999999); +INSERT INTO boston VALUES (484, 2.8183799999999999, 0, 18.100000000000001, 0, 0.53200000000000003, 5.7619999999999996, 40.299999999999997, 4.0983000000000001, 24, 666, 20.199999999999999, 392.92000000000002, 10.42, 21.800000000000001); +INSERT INTO boston VALUES (488, 4.8356700000000004, 0, 18.100000000000001, 0, 0.58299999999999996, 5.9050000000000002, 53.200000000000003, 3.1522999999999999, 24, 666, 20.199999999999999, 388.22000000000003, 11.449999999999999, 20.600000000000001); +INSERT INTO boston VALUES (492, 0.10574, 0, 27.739999999999998, 0, 0.60899999999999999, 5.9829999999999997, 98.799999999999997, 1.8681000000000001, 4, 711, 20.100000000000001, 390.11000000000001, 18.07, 13.6); +INSERT INTO boston VALUES (496, 0.17899000000000001, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.6699999999999999, 28.800000000000001, 2.7986, 6, 391, 19.199999999999999, 393.29000000000002, 17.600000000000001, 23.100000000000001); +INSERT INTO boston VALUES (500, 0.17782999999999999, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.569, 73.5, 2.3999000000000001, 6, 391, 19.199999999999999, 395.76999999999998, 15.1, 17.5); +INSERT INTO boston VALUES (504, 0.060760000000000002, 0, 11.93, 0, 0.57299999999999995, 6.976, 91, 2.1675, 1, 273, 21, 396.89999999999998, 5.6399999999999997, 23.899999999999999); +INSERT INTO boston VALUES (471, 4.3487900000000002, 0, 18.100000000000001, 0, 0.57999999999999996, 6.1669999999999998, 84, 3.0333999999999999, 24, 666, 20.199999999999999, 396.89999999999998, 16.289999999999999, 19.899999999999999); +INSERT INTO boston VALUES (475, 8.05579, 0, 18.100000000000001, 0, 0.58399999999999996, 5.4269999999999996, 95.400000000000006, 2.4298000000000002, 24, 666, 20.199999999999999, 352.57999999999998, 18.140000000000001, 13.800000000000001); +INSERT INTO boston VALUES (479, 10.233000000000001, 0, 18.100000000000001, 0, 0.61399999999999999, 6.1849999999999996, 96.700000000000003, 2.1705000000000001, 24, 666, 20.199999999999999, 379.69999999999999, 18.030000000000001, 14.6); +INSERT INTO boston VALUES (483, 5.73116, 0, 18.100000000000001, 0, 0.53200000000000003, 7.0609999999999999, 77, 3.4106000000000001, 24, 666, 20.199999999999999, 395.27999999999997, 7.0099999999999998, 25); +INSERT INTO boston VALUES (487, 5.6917499999999999, 0, 18.100000000000001, 0, 0.58299999999999996, 6.1139999999999999, 79.799999999999997, 3.5459000000000001, 24, 666, 20.199999999999999, 392.68000000000001, 14.98, 19.100000000000001); +INSERT INTO boston VALUES (491, 0.20746000000000001, 0, 27.739999999999998, 0, 0.60899999999999999, 5.093, 98, 1.8226, 4, 711, 20.100000000000001, 318.43000000000001, 29.68, 8.0999999999999996); +INSERT INTO boston VALUES (495, 0.27956999999999999, 0, 9.6899999999999995, 0, 0.58499999999999996, 5.9260000000000002, 42.600000000000001, 2.3816999999999999, 6, 391, 19.199999999999999, 396.89999999999998, 13.59, 24.5); +INSERT INTO boston VALUES (499, 0.23912, 0, 9.6899999999999995, 0, 0.58499999999999996, 6.0190000000000001, 65.299999999999997, 2.4091, 6, 391, 19.199999999999999, 396.89999999999998, 12.92, 21.199999999999999); +INSERT INTO boston VALUES (503, 0.045269999999999998, 0, 11.93, 0, 0.57299999999999995, 6.1200000000000001, 76.700000000000003, 2.2875000000000001, 1, 273, 21, 396.89999999999998, 9.0800000000000001, 20.600000000000001); + + +-------------- data insert finish ------------- + +select MADLIB_SCHEMA.xgboost_sk_Regressor('boston', 'boston_model_xgbr', 'id', 'target', 'crim, zn, Indus, chas, nox, rm, "Age"', NULL, $${'booster': ['gbtree'], 'eta': (0.1), 'max_depth': (5,6,7), 'objective':['reg:linear']}$$, TRUE, TRUE); + +select count(*) from boston_model_xgbr; +select id, train_timestamp, source_table, y_type, metrics, features, params from boston_model_xgbr; + +SELECT assert(count(*) = 3, 'xgboost: Xgboost model output has incorrect number of rows') +FROM (select * from boston_model_xgbr) q; + +drop table boston_model_xgbr; +select MADLIB_SCHEMA.xgboost_sk_Regressor('boston', 'boston_model_xgbr', 'id', 'target', 'crim, zn, Indus, chas, nox, rm, "Age"', NULL, + $${'booster': ['gbtree'], 'eta': (0.1), 'max_depth': (5,6,7), 'objective':['reg:linear']}$$); + +select id, train_timestamp, source_table, y_type, metrics, features, params from boston_model_xgbr; +select assert(metrics = 'evalution is OFF, no metrics', 'xgboost metrics error') from boston_model_xgbr; + +drop table if exists boston_model_xgbr3; +create table boston_model_xgbr3 as select * from boston_model_xgbr where id=3; + +select MADLIB_SCHEMA.xgboost_sk_predict('boston', 'boston_model_xgbr', 'boston_output_xgbr', 'id'); + +drop table if exists boston_xgb_results; +create table boston_xgb_results as select t1.id, prediction, target from boston_output_xgbr as t1, boston as t2 order by t1.id; + +select mean_squared_error('boston_xgb_results', 'table_out','prediction', 'target'); + +select * from table_out; \ No newline at end of file diff --git a/src/ports/postgres/modules/xgboost_gs/test/xgboost_test2.sql_in b/src/ports/postgres/modules/xgboost_gs/test/xgboost_test2.sql_in new file mode 100644 index 0000000..b3e3a09 --- /dev/null +++ b/src/ports/postgres/modules/xgboost_gs/test/xgboost_test2.sql_in @@ -0,0 +1,464 @@ + +DROP TABLE IF EXISTS telecom; +CREATE TABLE telecom( +"customerID" serial, gender VARCHAR(8), "SeniorCitizen" BOOLEAN, "Partner" BOOLEAN, "Dependents" BOOLEAN, +tenure INT, "PhoneService" VARCHAR(20), "MultipleLines" VARCHAR(20), "InternetService" VARCHAR(20), "OnlineSecurity" VARCHAR(20), +"OnlineBackup" VARCHAR(20), "DeviceProtection" VARCHAR(20), "TechSupport" VARCHAR(20), "StreamingTV" VARCHAR(20), "StreamingMovies" VARCHAR(20), +"Contract" VARCHAR(30), "PaperlessBilling" BOOLEAN, "PaymentMethod" VARCHAR(40), "MonthlyCharges" FLOAT DEFAULT 0, "TotalCharges" FLOAT DEFAULT 0, "Churn" INT DEFAULT 0); + +COPY telecom (gender, "SeniorCitizen", "Partner", "Dependents", tenure, "PhoneService", "MultipleLines", "InternetService", "OnlineSecurity", "OnlineBackup", "DeviceProtection", "TechSupport", "StreamingTV", "StreamingMovies", "Contract", "PaperlessBilling", "PaymentMethod", "MonthlyCharges", "TotalCharges", "Churn") FROM STDIN DELIMITER ','; +Male,0,0,0,2,1,0,DSL,1,1,0,0,0,0,Month-to-month,1,Mailed check,53.85,108.15,1 +Male,0,0,0,45,0,0 phone service,DSL,1,0,1,1,0,0,One year,0,Bank transfer (automatic),42.3,1840.75,0 +Female,0,0,0,2,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,70.7,151.65,1 +Female,0,0,0,8,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,99.65,820.5,1 +Male,0,0,1,22,1,1,Fiber optic,0,1,0,0,1,0,Month-to-month,1,Credit card (automatic),89.1,1949.4,0 +Female,0,0,0,10,0,0 phone service,DSL,1,0,0,0,0,0,Month-to-month,0,Mailed check,29.75,301.9,0 +Female,0,1,0,28,1,1,Fiber optic,0,0,1,1,1,1,Month-to-month,1,Electronic check,104.8,3046.05,1 +Male,0,0,1,62,1,0,DSL,1,1,0,0,0,0,One year,0,Bank transfer (automatic),56.15,3487.95,0 +Male,0,1,1,13,1,0,DSL,1,0,0,0,0,0,Month-to-month,1,Mailed check,49.95,587.45,0 +Male,0,0,0,16,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),18.95,326.8,0 +Male,0,1,0,58,1,1,Fiber optic,0,0,1,0,1,1,One year,0,Credit card (automatic),100.35,5681.1,0 +Male,0,0,0,49,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Bank transfer (automatic),103.7,5036.3,1 +Male,0,0,0,25,1,0,Fiber optic,1,0,1,1,1,1,Month-to-month,1,Electronic check,105.5,2686.05,0 +Female,0,1,1,69,1,1,Fiber optic,1,1,1,1,1,1,Two year,0,Credit card (automatic),113.25,7895.15,0 +Female,0,0,0,52,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.65,1022.95,0 +Male,0,0,1,71,1,1,Fiber optic,1,0,1,0,1,1,Two year,0,Bank transfer (automatic),106.7,7382.25,0 +Female,0,1,1,10,1,0,DSL,0,0,1,1,0,0,Month-to-month,0,Credit card (automatic),55.2,528.35,1 +Female,0,0,0,21,1,0,Fiber optic,0,1,1,0,0,1,Month-to-month,1,Electronic check,90.05,1862.9,0 +Male,1,0,0,1,0,0 phone service,DSL,0,0,1,0,0,1,Month-to-month,1,Electronic check,39.65,39.65,1 +Male,0,1,0,12,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Bank transfer (automatic),19.8,202.25,0 +Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.15,20.15,1 +Female,0,1,0,58,1,1,DSL,0,1,0,1,0,0,Two year,1,Credit card (automatic),59.9,3505.1,0 +Male,0,1,1,49,1,0,DSL,1,1,0,1,0,0,Month-to-month,0,Credit card (automatic),59.6,2970.3,0 +Female,0,0,0,30,1,0,DSL,1,1,0,0,0,0,Month-to-month,1,Bank transfer (automatic),55.3,1530.6,0 +Male,0,1,1,47,1,1,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Electronic check,99.35,4749.15,1 +Male,0,1,1,1,0,0 phone service,DSL,0,1,0,0,0,0,Month-to-month,0,Electronic check,30.2,30.2,1 +Male,0,1,0,72,1,1,DSL,1,1,1,1,1,1,Two year,1,Credit card (automatic),90.25,6369.45,0 +Female,0,0,1,17,1,0,DSL,0,0,0,0,1,1,Month-to-month,1,Mailed check,64.7,1093.1,1 +Female,1,1,0,71,1,1,Fiber optic,1,1,1,1,0,0,Two year,1,Credit card (automatic),96.35,6766.95,0 +Male,1,1,0,2,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Credit card (automatic),95.5,181.65,0 +Female,0,1,1,27,1,0,DSL,1,1,1,1,0,0,One year,0,Mailed check,66.15,1874.45,0 +Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Bank transfer (automatic),20.2,20.2,0 +Male,1,0,0,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,0,Bank transfer (automatic),45.25,45.25,0 +Female,0,1,1,72,1,1,Fiber optic,1,1,0,1,1,0,Two year,0,Bank transfer (automatic),99.9,7251.7,0 +Male,0,0,0,5,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,69.7,316.9,1 +Female,0,0,0,46,1,0,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Credit card (automatic),74.8,3548.3,0 +Male,0,0,0,34,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,106.35,3549.25,1 +Female,0,0,0,11,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Bank transfer (automatic),97.85,1105.4,1 +Male,0,1,1,10,1,0,DSL,0,1,0,0,0,0,One year,0,Mailed check,49.55,475.7,0 +Female,0,1,1,70,1,1,DSL,1,1,0,0,1,0,Two year,1,Credit card (automatic),69.2,4872.35,0 +Female,0,1,1,17,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.75,418.25,0 +Female,0,0,0,63,1,1,DSL,1,1,1,1,1,0,Two year,1,Credit card (automatic),79.85,4861.45,0 +Female,0,1,0,13,1,1,DSL,1,1,0,1,1,0,Month-to-month,1,Electronic check,76.2,981.45,0 +Female,0,0,0,49,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,1,Electronic check,84.5,3906.7,0 +Male,0,0,0,2,1,0,DSL,0,1,0,0,0,0,Month-to-month,0,Mailed check,49.25,97,0 +Female,0,0,0,2,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Electronic check,80.65,144.15,1 +Male,0,0,0,52,1,1,DSL,1,0,0,1,1,1,Two year,1,Credit card (automatic),79.75,4217.8,0 +Female,0,1,1,69,1,1,DSL,1,0,1,1,0,0,Two year,1,Credit card (automatic),64.15,4254.1,0 +Female,1,0,0,43,1,1,Fiber optic,0,1,0,0,1,0,Month-to-month,1,Electronic check,90.25,3838.75,0 +Female,0,0,0,15,1,0,Fiber optic,1,1,0,0,1,1,Month-to-month,1,Credit card (automatic),99.1,1426.4,1 +Female,1,1,0,25,1,1,DSL,1,0,0,1,1,0,Month-to-month,1,Credit card (automatic),69.5,1752.65,0 +Female,1,1,0,8,1,1,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Credit card (automatic),80.65,633.3,1 +Female,1,1,1,60,1,0,DSL,1,1,1,1,0,1,One year,1,Credit card (automatic),74.85,4456.35,0 +Male,1,0,0,18,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,95.45,1752.55,1 +Female,0,1,1,63,1,1,Fiber optic,1,0,0,0,1,1,One year,1,Credit card (automatic),99.65,6311.2,0 +Male,1,1,1,66,1,1,Fiber optic,0,1,1,1,1,1,One year,1,Electronic check,108.45,7076.35,0 +Female,0,1,1,34,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),24.95,894.3,0 +Female,0,0,0,72,1,1,Fiber optic,0,0,1,1,1,1,Two year,1,Credit card (automatic),107.5,7853.7,0 +Female,0,1,0,47,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,100.5,4707.1,0 +Male,0,0,0,60,1,1,Fiber optic,0,1,0,0,1,0,Month-to-month,1,Bank transfer (automatic),89.9,5450.7,0 +Male,0,1,0,72,0,0 phone service,DSL,1,1,1,0,0,0,Two year,0,Bank transfer (automatic),42.1,2962,0 +Female,0,1,1,18,1,0,DSL,0,0,1,1,0,0,One year,1,Credit card (automatic),54.4,957.1,0 +Female,0,0,0,9,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,0,Electronic check,94.4,857.25,1 +Female,0,0,0,3,1,0,DSL,0,1,0,1,1,1,Month-to-month,1,Electronic check,75.3,244.1,0 +Male,0,1,0,47,1,1,Fiber optic,0,1,0,0,0,0,One year,1,Electronic check,78.9,3650.35,0 +Female,0,0,0,31,1,0,DSL,0,1,1,1,1,1,Two year,0,Mailed check,79.2,2497.2,0 +Female,0,1,1,50,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),20.15,930.9,0 +Male,0,0,0,10,1,0,Fiber optic,1,0,1,0,0,0,Month-to-month,1,Mailed check,79.85,887.35,0 +Male,0,0,0,1,1,0,DSL,0,0,0,1,0,0,Month-to-month,0,Mailed check,49.05,49.05,0 +Female,0,1,1,52,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Electronic check,20.4,1090.65,0 +Male,1,1,1,64,1,1,Fiber optic,1,0,1,1,1,1,Two year,1,Bank transfer (automatic),111.6,7099,0 +Male,0,1,1,62,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),24.25,1424.6,0 +Female,0,0,1,3,1,0,DSL,1,0,0,1,0,1,Month-to-month,1,Bank transfer (automatic),64.5,177.4,0 +Female,1,0,0,56,1,1,Fiber optic,1,1,1,0,1,1,One year,0,Electronic check,110.5,6139.5,0 +Female,0,0,0,46,1,0,DSL,0,0,0,0,0,1,One year,0,Credit card (automatic),55.65,2688.85,0 +Female,0,1,1,8,1,0,DSL,1,1,0,0,0,0,Month-to-month,0,Mailed check,54.65,482.25,0 +Male,1,0,0,30,1,0,DSL,1,1,0,0,1,1,Month-to-month,1,Electronic check,74.75,2111.3,0 +Female,0,1,1,45,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Credit card (automatic),25.9,1216.6,0 +Female,0,0,1,1,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Electronic check,79.35,79.35,1 +Female,0,1,1,11,0,0 phone service,DSL,1,0,0,0,1,1,Month-to-month,0,Electronic check,50.55,565.35,0 +Female,0,1,0,7,1,0,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Bank transfer (automatic),75.15,496.9,1 +Female,0,0,0,42,1,0,Fiber optic,0,1,1,1,1,1,Month-to-month,1,Bank transfer (automatic),103.8,4327.5,0 +Female,0,1,0,49,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),20.15,973.35,0 +Male,0,0,0,9,1,1,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Credit card (automatic),99.3,918.75,0 +Female,0,1,0,35,1,0,DSL,1,0,0,0,1,0,One year,1,Bank transfer (automatic),62.15,2215.45,0 +Female,0,1,1,48,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),20.65,1057,0 +Female,0,1,1,46,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,19.95,927.1,0 +Male,0,1,0,29,0,0 phone service,DSL,0,0,0,0,1,0,Month-to-month,0,Mailed check,33.75,1009.25,0 +Male,0,1,1,30,1,0,Fiber optic,0,1,1,0,0,0,Month-to-month,0,Bank transfer (automatic),82.05,2570.2,0 +Male,1,0,0,1,1,0,Fiber optic,0,0,0,1,0,0,Month-to-month,0,Electronic check,74.7,74.7,0 +Male,0,1,1,66,1,1,DSL,1,0,1,1,1,1,Two year,1,Mailed check,84,5714.25,0 +Female,0,0,0,65,1,1,Fiber optic,1,1,1,0,1,1,Month-to-month,1,Credit card (automatic),111.05,7107,0 +Male,0,0,0,72,1,1,Fiber optic,0,1,1,0,1,1,Two year,1,Bank transfer (automatic),100.9,7459.05,0 +Female,0,0,0,12,1,1,Fiber optic,1,0,0,0,0,0,Month-to-month,1,Electronic check,78.95,927.35,1 +Male,0,1,1,71,1,1,DSL,1,1,0,1,0,0,One year,1,Credit card (automatic),66.85,4748.7,0 +Male,0,0,0,5,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,21.05,113.85,1 +Male,0,0,0,52,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),21,1107.2,0 +Female,1,1,0,25,1,0,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,98.5,2514.5,1 +Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.2,20.2,0 +Female,0,1,1,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Electronic check,19.45,19.45,0 +Male,0,0,0,38,1,1,Fiber optic,0,0,1,1,1,0,One year,0,Bank transfer (automatic),95,3605.6,0 +Female,1,1,0,66,0,0 phone service,DSL,0,1,1,0,1,0,One year,0,Bank transfer (automatic),45.55,3027.25,0 +Male,0,1,0,68,1,1,Fiber optic,0,1,1,1,1,1,Two year,1,Bank transfer (automatic),110,7611.85,1 +Male,0,0,0,5,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,0,Mailed check,24.3,100.2,0 +Female,0,1,1,72,1,1,Fiber optic,1,1,0,0,1,1,One year,1,Electronic check,104.15,7303.05,0 +Female,0,0,0,32,0,0 phone service,DSL,1,0,0,0,0,0,One year,0,Mailed check,30.15,927.65,0 +Male,0,0,0,43,1,1,Fiber optic,0,0,0,0,1,1,One year,0,Mailed check,94.35,3921.3,0 +Male,0,1,1,72,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),19.4,1363.25,0 +Male,0,1,0,55,1,1,Fiber optic,1,1,0,0,1,0,Month-to-month,1,Electronic check,96.75,5238.9,1 +Female,0,0,0,52,1,0,DSL,1,0,1,1,0,0,One year,0,Credit card (automatic),57.95,3042.25,0 +Female,0,0,0,43,1,0,Fiber optic,0,1,1,0,1,0,Month-to-month,1,Electronic check,91.65,3954.1,0 +Female,1,1,0,37,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,76.5,2868.15,1 +Female,0,1,1,64,0,0 phone service,DSL,0,1,0,1,1,1,Two year,1,Electronic check,54.6,3423.5,0 +Male,0,1,1,3,1,0,Fiber optic,1,1,0,0,1,0,Month-to-month,0,Electronic check,89.85,248.4,0 +Female,0,0,0,36,0,0 phone service,DSL,1,0,0,0,0,0,Two year,0,Bank transfer (automatic),31.05,1126.35,0 +Female,0,1,1,10,1,1,Fiber optic,1,0,0,0,1,1,Month-to-month,1,Electronic check,100.25,1064.65,1 +Female,0,0,0,41,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,20.65,835.15,0 +Male,0,1,1,27,1,1,Fiber optic,0,1,0,1,0,0,Month-to-month,1,Credit card (automatic),85.2,2151.6,0 +Female,0,1,1,56,1,1,Fiber optic,0,0,1,0,1,1,One year,1,Credit card (automatic),99.8,5515.45,0 +Female,0,0,0,6,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.7,112.75,0 +Male,0,0,0,3,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,74.4,229.55,1 +Female,0,1,1,7,1,0,DSL,1,0,0,0,0,0,Month-to-month,0,Mailed check,50.7,350.35,0 +Female,0,1,1,4,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.85,62.9,0 +Male,0,0,0,33,1,0,Fiber optic,1,0,0,1,1,0,Two year,1,Electronic check,88.95,3027.65,0 +Female,1,0,0,27,1,1,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Electronic check,78.05,2135.5,1 +Male,0,1,0,72,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),23.55,1723.95,0 +Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.75,19.75,0 +Male,1,0,0,71,0,0 phone service,DSL,1,1,0,0,1,1,One year,1,Electronic check,56.45,3985.35,0 +Female,0,0,0,13,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,1,Electronic check,85.95,1215.65,0 +Female,0,1,1,25,0,0 phone service,DSL,1,1,1,0,1,1,Month-to-month,1,Credit card (automatic),58.6,1502.65,1 +Male,0,0,0,67,1,0,DSL,0,0,0,1,0,0,Two year,0,Bank transfer (automatic),50.55,3260.1,0 +Male,0,0,0,1,0,0 phone service,DSL,0,0,0,0,0,1,Month-to-month,1,Electronic check,35.45,35.45,1 +Male,0,0,0,2,1,0,DSL,0,0,0,0,0,0,Month-to-month,0,Electronic check,44.35,81.25,1 +Female,0,0,0,43,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Credit card (automatic),25.7,1188.2,0 +Female,0,0,0,23,1,0,Fiber optic,1,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),75,1778.5,0 +Female,0,1,1,64,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,20.2,1277.75,0 +Male,0,0,1,57,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,19.6,1170.55,0 +Female,1,1,0,1,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Mailed check,70.45,70.45,1 +Female,1,1,1,72,1,1,DSL,1,1,1,1,1,1,Two year,1,Credit card (automatic),88.05,6425.65,0 +Female,0,0,0,8,1,1,DSL,1,0,0,1,0,1,Month-to-month,0,Electronic check,71.15,563.65,1 +Female,0,1,0,61,1,1,Fiber optic,0,1,0,0,1,1,Two year,1,Bank transfer (automatic),101.05,5971.25,0 +Male,0,0,0,64,1,1,Fiber optic,1,1,0,0,0,0,One year,1,Credit card (automatic),84.3,5289.05,0 +Male,1,1,0,71,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),23.95,1756.2,0 +Female,0,1,1,65,1,1,Fiber optic,1,1,0,1,1,0,Two year,1,Credit card (automatic),99.05,6416.7,0 +Male,0,0,0,3,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,19.6,61.35,0 +Male,0,0,0,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,45.65,45.65,1 +Male,0,0,1,30,1,0,DSL,0,1,1,0,1,0,One year,0,Credit card (automatic),64.5,1929.95,0 +Male,0,1,1,15,1,0,DSL,0,1,1,1,0,1,Month-to-month,1,Mailed check,69.5,1071.4,0 +Female,0,1,1,8,1,0,DSL,0,0,0,1,1,1,Month-to-month,1,Electronic check,68.55,564.35,0 +Male,0,0,0,7,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,0,Mailed check,95,655.5,1 +Female,0,1,1,70,1,1,Fiber optic,1,1,1,0,1,1,One year,1,Credit card (automatic),108.15,7930.55,0 +Male,0,1,1,62,1,0,DSL,1,1,1,1,1,1,Two year,0,Electronic check,86.1,5215.25,0 +Female,0,1,1,6,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.7,113.5,0 +Female,0,1,1,14,1,0,Fiber optic,0,0,0,0,0,1,Month-to-month,1,Bank transfer (automatic),80.9,1152.8,0 +Female,0,0,0,22,1,1,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Bank transfer (automatic),84.15,1821.95,0 +Male,0,1,1,22,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.15,419.9,0 +Male,0,1,1,16,1,0,DSL,1,0,1,0,1,0,Two year,0,Mailed check,64.25,1024,0 +Male,0,0,0,10,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),25.7,251.6,0 +Female,0,0,1,13,1,0,DSL,1,1,0,0,0,0,Month-to-month,1,Electronic check,56,764.55,0 +Female,0,1,0,20,1,1,Fiber optic,1,0,0,0,0,0,Month-to-month,0,Electronic check,82.4,1592.35,1 +Female,0,0,0,2,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,69.7,135.2,0 +Male,0,0,0,53,1,1,DSL,0,1,1,1,0,1,Two year,1,Bank transfer (automatic),73.9,3958.25,0 +Female,0,1,1,11,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.6,233.9,0 +Male,0,1,0,69,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,19.9,1363.45,0 +Female,0,0,0,4,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),70.9,273,1 +Male,0,1,0,72,1,1,DSL,1,1,1,1,1,1,Two year,1,Bank transfer (automatic),89.05,6254.45,0 +Male,1,1,0,58,0,0 phone service,DSL,0,1,1,0,0,1,Month-to-month,1,Electronic check,45.3,2651.2,1 +Female,0,1,1,16,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Electronic check,20.4,321.4,0 +Male,0,1,0,43,1,0,Fiber optic,1,0,1,1,0,0,One year,1,Bank transfer (automatic),84.25,3539.25,0 +Female,0,1,0,2,1,1,Fiber optic,0,0,1,1,1,1,Month-to-month,1,Electronic check,104.4,242.8,1 +Male,0,1,0,14,1,0,DSL,0,1,1,1,1,1,Two year,0,Mailed check,81.95,1181.75,0 +Female,0,1,0,53,1,0,Fiber optic,0,0,1,0,1,1,One year,1,Bank transfer (automatic),94.85,5000.2,1 +Female,0,0,0,32,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Bank transfer (automatic),20.55,654.55,0 +Female,0,1,0,34,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Electronic check,24.7,780.2,0 +Female,1,0,0,15,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Mailed check,74.45,1145.7,1 +Female,1,0,0,7,1,0,Fiber optic,1,0,0,0,0,0,Month-to-month,0,Bank transfer (automatic),76.45,503.6,1 +Female,0,1,1,15,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Bank transfer (automatic),105.35,1559.25,0 +Male,0,1,0,61,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Credit card (automatic),20.55,1252,0 +Female,0,0,0,1,0,0 phone service,DSL,0,1,0,0,0,0,Month-to-month,0,Bank transfer (automatic),29.95,29.95,1 +Female,0,0,0,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,0,Mailed check,45.3,45.3,0 +Male,0,0,0,8,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Bank transfer (automatic),84.5,662.65,1 +Male,0,1,1,33,1,0,DSL,1,0,1,0,1,1,One year,0,Credit card (automatic),74.75,2453.3,0 +Female,0,0,0,13,1,0,Fiber optic,0,1,1,0,0,0,Month-to-month,1,Electronic check,79.25,1111.65,1 +Female,0,1,0,1,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,24.8,24.8,1 +Male,0,0,0,20,1,0,DSL,0,1,0,0,0,0,Month-to-month,1,Mailed check,51.8,1023.85,0 +Male,0,0,0,3,0,0 phone service,DSL,0,0,0,1,0,0,Month-to-month,0,Electronic check,30.4,82.15,0 +Female,0,0,0,13,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Electronic check,19.65,244.8,0 +Female,0,1,0,40,1,1,DSL,0,1,0,0,0,0,Month-to-month,0,Credit card (automatic),56.6,2379.1,0 +Male,0,1,1,43,1,1,DSL,1,0,0,1,0,1,One year,1,Credit card (automatic),71.9,3173.35,0 +Male,0,1,0,6,1,0,Fiber optic,0,1,1,0,0,1,Month-to-month,1,Electronic check,91,531,1 +Female,0,1,1,69,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),19.75,1375.4,0 +Female,0,1,1,72,1,1,Fiber optic,1,1,1,0,1,1,Two year,0,Credit card (automatic),109.7,8129.3,0 +Male,0,1,1,59,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),19.3,1192.7,0 +Female,0,1,0,20,1,0,Fiber optic,1,1,0,1,1,0,Month-to-month,1,Electronic check,96.55,1901.65,0 +Male,0,1,1,24,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),24.1,587.4,0 +Male,0,0,0,59,1,1,Fiber optic,0,1,1,1,1,1,Month-to-month,1,Electronic check,111.35,6519.75,0 +Male,0,1,1,72,1,1,Fiber optic,0,1,1,1,1,1,Two year,1,Bank transfer (automatic),112.25,8041.65,0 +Male,0,0,1,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.75,20.75,0 +Female,0,1,0,27,1,0,Fiber optic,1,0,0,1,1,1,One year,1,Credit card (automatic),101.9,2681.15,0 +Female,0,0,0,14,1,0,Fiber optic,0,1,0,1,0,0,Month-to-month,1,Credit card (automatic),80.05,1112.3,0 +Male,0,1,1,71,1,1,Fiber optic,1,0,1,0,1,1,Two year,0,Electronic check,105.55,7405.5,0 +Male,0,0,1,13,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Credit card (automatic),78.3,1033.95,0 +Male,0,0,0,44,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Credit card (automatic),68.85,2958.95,0 +Female,0,0,0,33,1,0,Fiber optic,0,1,0,1,0,0,One year,1,Bank transfer (automatic),79.95,2684.85,0 +Male,0,1,1,72,0,0 phone service,DSL,1,1,1,1,0,1,Two year,1,Credit card (automatic),55.45,4179.2,0 +Male,0,0,0,1,1,0,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Electronic check,79.9,79.9,1 +Female,0,0,0,19,1,0,Fiber optic,1,0,1,1,1,1,Month-to-month,1,Electronic check,106.6,1934.45,1 +Male,0,1,0,64,1,0,Fiber optic,0,1,1,1,1,1,One year,1,Credit card (automatic),102.45,6654.1,0 +Male,0,0,0,2,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Credit card (automatic),46,84.5,1 +Female,0,0,0,1,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,0,Electronic check,25.25,25.25,0 +Male,0,0,1,61,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,19.75,1124.2,0 +Female,0,1,1,29,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Electronic check,20,540.05,0 +Male,1,1,0,23,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Electronic check,86.8,1975.85,0 +Female,0,1,0,57,0,0 phone service,DSL,1,1,1,0,1,1,Month-to-month,1,Bank transfer (automatic),58.75,3437.45,0 +Male,0,1,1,72,0,0 phone service,DSL,1,1,1,1,0,0,Two year,0,Credit card (automatic),45.25,3139.8,0 +Male,0,1,0,66,0,0 phone service,DSL,1,1,1,1,1,0,Two year,1,Bank transfer (automatic),56.6,3789.2,0 +Male,0,1,1,65,1,1,DSL,0,1,1,1,1,1,Two year,1,Bank transfer (automatic),84.2,5324.5,0 +Female,0,0,0,8,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Bank transfer (automatic),80,624.6,0 +Female,0,0,0,4,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,70.15,268.35,1 +Female,0,1,0,71,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),24.75,1836.9,0 +Male,0,1,1,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Electronic check,20.2,20.2,0 +Male,0,1,0,4,1,0,DSL,0,0,0,1,0,0,Month-to-month,0,Mailed check,50.05,179.35,1 +Female,0,0,0,12,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Bank transfer (automatic),19.35,219.35,0 +Male,0,0,0,24,0,0 phone service,DSL,1,0,0,0,1,1,One year,1,Mailed check,50.6,1288.75,0 +Female,0,1,1,31,1,0,DSL,1,0,1,1,1,1,One year,1,Mailed check,81.15,2545.75,0 +Female,0,1,0,1,1,0,DSL,0,0,0,0,1,0,Month-to-month,0,Mailed check,55.2,55.2,1 +Male,0,0,0,30,1,1,Fiber optic,0,0,0,1,0,1,Month-to-month,1,Credit card (automatic),89.9,2723.15,0 +Female,0,1,1,47,1,1,DSL,1,1,0,1,1,1,Two year,1,Credit card (automatic),85.3,4107.25,0 +Male,0,0,0,54,1,1,Fiber optic,1,1,0,1,1,1,Two year,1,Credit card (automatic),108,5760.65,0 +Male,0,0,0,50,1,0,Fiber optic,1,1,0,1,0,1,Month-to-month,1,Credit card (automatic),93.5,4747.5,0 +Male,0,0,0,1,1,0,Fiber optic,1,0,0,0,0,1,Month-to-month,1,Mailed check,84.6,84.6,1 +Female,0,0,0,72,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Credit card (automatic),20.25,1566.9,0 +Female,0,0,0,29,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Electronic check,25.15,702,0 +Male,0,0,0,2,1,1,DSL,0,1,0,0,0,0,Month-to-month,1,Mailed check,54.4,114.1,1 +Female,0,0,0,10,0,0 phone service,DSL,0,0,0,1,0,0,Two year,1,Mailed check,29.6,299.05,0 +Male,0,1,0,18,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,73.15,1305.95,0 +Female,1,0,0,11,1,0,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Electronic check,95,1120.3,1 +Male,0,0,0,16,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,19.75,284.35,0 +Female,0,0,0,72,1,1,DSL,0,1,1,1,1,1,Two year,0,Bank transfer (automatic),86.6,6350.5,0 +Male,0,0,0,72,1,1,Fiber optic,0,1,1,1,1,1,Two year,0,Bank transfer (automatic),109.2,7878.3,0 +Female,0,1,1,41,1,0,DSL,1,0,0,1,1,1,One year,1,Credit card (automatic),74.7,3187.65,0 +Female,1,1,0,65,1,0,Fiber optic,1,1,1,0,0,1,Month-to-month,0,Electronic check,94.4,6126.15,0 +Female,1,0,0,13,1,0,DSL,0,1,1,0,0,0,Month-to-month,1,Electronic check,54.8,731.3,0 +Male,1,0,0,4,1,0,Fiber optic,1,0,0,0,0,0,Month-to-month,0,Electronic check,75.35,273.4,0 +Male,0,0,0,41,1,1,DSL,0,0,1,0,0,1,One year,0,Mailed check,65,2531.8,0 +Male,1,0,0,15,1,0,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Electronic check,74.4,1074.3,1 +Male,0,0,0,1,1,0,DSL,1,0,0,0,0,0,Month-to-month,1,Mailed check,48.55,48.55,1 +Male,0,0,0,42,1,1,Fiber optic,1,1,1,0,0,1,One year,1,Electronic check,99,4298.45,0 +Female,0,1,0,51,1,0,Fiber optic,1,1,1,0,1,0,One year,1,Electronic check,93.5,4619.55,0 +Female,0,1,1,2,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Electronic check,70.4,147.15,1 +Male,1,1,0,1,0,0 phone service,DSL,0,1,0,0,1,0,Month-to-month,0,Electronic check,40.2,40.2,1 +Male,0,0,1,32,1,1,DSL,0,1,1,1,1,1,One year,0,Credit card (automatic),83.7,2633.3,0 +Male,0,0,0,10,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,19.85,193.05,0 +Female,0,1,1,67,0,0 phone service,DSL,0,1,1,1,1,1,Two year,1,Electronic check,59.55,4103.9,0 +Female,0,1,1,61,1,1,Fiber optic,1,1,1,1,1,1,Two year,0,Credit card (automatic),115.1,7008.15,0 +Male,0,0,0,50,1,1,Fiber optic,1,1,1,1,1,1,One year,0,Credit card (automatic),114.35,5791.1,0 +Female,0,1,0,2,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Credit card (automatic),44.6,80.55,1 +Female,1,1,0,29,0,0 phone service,DSL,1,0,1,0,0,1,Month-to-month,1,Electronic check,45,1228.65,0 +Male,1,0,0,3,0,0 phone service,DSL,0,0,1,0,0,1,Month-to-month,1,Electronic check,41.15,132.2,1 +Male,1,0,0,13,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,106.9,1364.3,1 +Male,1,1,0,57,1,1,Fiber optic,0,1,0,0,0,1,Month-to-month,1,Electronic check,89.85,4925.35,0 +Female,0,0,0,31,0,0 phone service,DSL,1,1,0,1,0,1,Two year,1,Bank transfer (automatic),49.85,1520.1,0 +Female,0,1,0,45,1,1,Fiber optic,1,1,1,1,1,1,One year,0,Mailed check,113.3,5032.25,0 +Female,0,1,1,61,1,1,DSL,1,1,1,1,1,1,Two year,0,Mailed check,88.1,5526.75,0 +Male,0,0,0,50,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),24.9,1195.25,0 +Female,1,0,0,19,1,0,Fiber optic,1,1,0,1,1,1,Month-to-month,1,Mailed check,105,2007.25,0 +Male,0,0,0,59,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,19.35,1099.6,1 +Female,0,1,0,71,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Credit card (automatic),24.25,1732.95,0 +Female,1,0,0,16,1,0,Fiber optic,0,0,0,1,1,1,Month-to-month,1,Electronic check,94.45,1511.2,1 +Male,0,1,0,57,1,0,DSL,1,1,0,1,0,0,Two year,1,Electronic check,59.75,3450.15,0 +Male,0,0,0,1,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,0,Bank transfer (automatic),24.8,24.8,1 +Male,1,1,0,20,1,1,Fiber optic,0,1,0,1,1,1,Month-to-month,0,Electronic check,107.05,2172.05,0 +Male,0,0,0,1,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Mailed check,70.6,70.6,1 +Male,0,1,0,5,1,1,Fiber optic,0,1,0,1,0,0,Month-to-month,1,Electronic check,85.4,401.1,1 +Female,0,1,0,52,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,105.05,5624.85,1 +Male,1,1,0,21,1,0,DSL,1,1,0,0,0,1,Month-to-month,1,Bank transfer (automatic),64.95,1339.8,0 +Male,0,0,0,14,0,0 phone service,DSL,0,0,1,1,1,1,Month-to-month,1,Electronic check,55,771.95,0 +Female,0,0,0,5,1,0,DSL,0,1,0,0,0,0,Month-to-month,0,Bank transfer (automatic),50.55,244.75,0 +Female,0,0,0,6,1,0,DSL,1,1,0,0,0,0,Month-to-month,0,Mailed check,55.15,322.9,0 +Female,0,0,0,10,1,0,DSL,0,1,0,0,0,0,Month-to-month,0,Electronic check,51.2,498.25,0 +Female,0,0,1,1,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,25.4,25.4,0 +Female,0,0,0,68,1,1,DSL,1,0,0,0,0,0,Month-to-month,0,Mailed check,54.45,3687.75,0 +Male,0,1,1,18,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,0,Mailed check,95.15,1779.95,1 +Female,0,0,0,22,1,1,Fiber optic,0,0,0,0,0,0,One year,1,Credit card (automatic),76,1783.6,0 +Female,0,0,0,20,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,44.35,927.15,0 +Male,0,1,0,1,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Electronic check,70,70,1 +Male,1,0,0,8,1,0,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Electronic check,74.5,606.55,1 +Male,0,0,0,10,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Credit card (automatic),44.85,435.4,1 +Male,1,0,0,24,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,76.1,1712.7,1 +Male,0,0,0,35,1,0,DSL,1,1,1,0,0,0,One year,0,Mailed check,61.2,2021.2,0 +Male,0,1,1,23,1,0,Fiber optic,0,1,1,1,0,0,One year,0,Mailed check,86.8,1940.8,0 +Female,1,0,0,6,1,0,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,89.35,567.8,0 +Female,0,0,0,12,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.7,220.35,0 +Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.25,20.25,0 +Female,0,1,1,71,1,0,Fiber optic,0,0,1,0,0,0,Two year,0,Electronic check,76.05,5436.45,0 +Male,1,1,0,35,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,100.8,3437.5,0 +Male,0,1,1,40,1,1,DSL,0,0,1,0,1,1,Month-to-month,1,Electronic check,74.55,3015.75,0 +Female,0,0,1,1,1,0,Fiber optic,1,0,0,0,0,0,Month-to-month,1,Mailed check,73.6,73.6,1 +Male,0,0,0,23,1,0,DSL,0,0,0,0,1,1,One year,1,Bank transfer (automatic),64.9,1509.8,0 +Female,1,0,0,4,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,95.45,396.1,1 +Male,0,0,0,4,1,0,Fiber optic,0,0,1,1,0,1,Month-to-month,1,Electronic check,90.4,356.65,0 +Female,0,1,1,68,0,0 phone service,DSL,0,1,1,1,1,1,Two year,0,Credit card (automatic),60.3,4109,0 +Male,0,0,1,38,1,1,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Mailed check,81.85,3141.7,0 +Male,0,1,1,52,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Mailed check,24.8,1229.1,0 +Male,1,1,0,32,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,74.9,2303.35,1 +Male,0,1,1,29,1,0,DSL,0,0,1,1,1,1,Two year,0,Mailed check,75.55,2054.4,0 +Male,1,1,1,38,1,1,Fiber optic,0,1,0,0,1,1,Month-to-month,0,Electronic check,101.15,3741.85,0 +Male,0,0,1,48,1,1,DSL,1,0,1,0,1,1,One year,1,Credit card (automatic),78.75,3682.45,0 +Female,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Bank transfer (automatic),19.25,19.25,0 +Female,0,0,0,22,1,0,Fiber optic,1,0,0,1,0,1,Month-to-month,1,Electronic check,89.05,1886.25,0 +Female,0,0,0,43,1,1,Fiber optic,1,1,1,1,1,1,Two year,0,Credit card (automatic),115.05,4895.1,0 +Female,1,1,0,5,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,69.35,341.6,0 +Male,0,0,0,5,1,0,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Electronic check,80.6,415.55,1 +Male,0,1,1,51,1,1,Fiber optic,1,1,0,1,1,1,One year,0,Credit card (automatic),110.05,5686.4,0 +Male,0,0,0,71,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Credit card (automatic),19.9,1355.1,0 +Female,0,1,1,38,1,0,DSL,1,1,0,1,1,1,One year,0,Credit card (automatic),80.3,3058.65,1 +Male,1,1,0,24,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,93.15,2231.05,1 +Male,0,1,0,35,1,1,Fiber optic,0,0,0,1,1,0,Month-to-month,1,Credit card (automatic),91.5,3236.35,0 +Female,1,0,0,54,1,1,DSL,1,1,0,1,1,1,One year,0,Bank transfer (automatic),82.45,4350.1,1 +Female,0,1,1,72,0,0 phone service,DSL,0,1,1,1,1,1,Two year,1,Electronic check,60,4264,0 +Male,0,0,0,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,44.8,44.8,0 +Female,0,1,1,9,1,0,DSL,1,0,0,0,0,0,One year,0,Mailed check,48.6,422.3,0 +Male,0,0,0,69,0,0 phone service,DSL,1,1,0,1,1,1,Two year,1,Bank transfer (automatic),60.05,4176.7,0 +Female,0,1,0,52,1,1,Fiber optic,0,1,0,1,1,1,Month-to-month,1,Credit card (automatic),102.7,5138.1,0 +Female,1,1,0,11,1,0,Fiber optic,0,0,0,1,1,0,Month-to-month,1,Electronic check,82.9,880.05,0 +Female,1,0,0,2,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,70.35,139.05,1 +Female,1,1,1,28,0,0 phone service,DSL,0,0,0,0,1,0,Month-to-month,1,Electronic check,35.9,973.65,0 +Female,1,0,0,17,1,1,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Credit card (automatic),82.65,1470.05,0 +Female,0,1,1,35,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,19.85,739.35,0 +Female,0,0,0,8,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Mailed check,19.2,161.95,0 +Male,0,0,0,46,1,1,Fiber optic,0,1,1,0,1,0,One year,1,Credit card (automatic),94.9,4422.95,0 +Female,0,0,0,7,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,73.85,511.25,1 +Female,0,0,0,2,1,1,Fiber optic,0,0,1,0,0,0,Month-to-month,0,Mailed check,80.6,155.8,1 +Male,0,1,1,68,1,1,DSL,0,1,1,1,0,1,One year,1,Bank transfer (automatic),75.8,5293.95,1 +Female,0,0,0,43,1,1,Fiber optic,0,1,1,0,1,1,Two year,1,Electronic check,104.6,4759.85,1 +Female,0,0,0,68,1,1,DSL,1,1,1,1,1,1,Two year,1,Electronic check,88.15,6148.45,0 +Female,0,0,0,36,1,0,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Credit card (automatic),94.8,3565.65,0 +Male,0,1,0,63,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,103.4,6603,1 +Female,1,0,0,32,1,0,DSL,0,1,0,1,0,0,Month-to-month,0,Bank transfer (automatic),54.65,1830.1,0 +Female,0,1,0,71,1,1,DSL,1,1,1,1,1,1,Two year,0,Credit card (automatic),85.75,6223.8,0 +Female,0,1,1,66,1,1,DSL,1,1,1,0,0,0,Month-to-month,1,Bank transfer (automatic),67.45,4508.65,0 +Male,0,0,0,63,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.5,1328.15,0 +Female,0,0,1,41,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Credit card (automatic),20.25,865,0 +Female,1,0,0,1,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,72.1,72.1,0 +Female,0,0,0,2,1,0,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,90.4,168.2,1 +Female,0,1,0,70,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,0,Bank transfer (automatic),19.45,1303.5,0 +Female,0,0,0,23,0,0 phone service,DSL,0,0,0,0,1,1,Month-to-month,1,Electronic check,44.95,996.85,0 +Female,1,1,0,64,1,0,Fiber optic,1,1,1,0,0,1,One year,0,Bank transfer (automatic),97,6430.9,0 +Male,0,1,1,37,1,1,DSL,1,0,1,1,0,0,Two year,1,Credit card (automatic),62.8,2278.75,0 +Male,0,0,1,17,1,0,DSL,0,0,0,0,0,0,One year,0,Mailed check,44.6,681.4,0 +Female,1,0,0,7,1,0,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,89.15,574.35,0 +Male,0,1,1,4,1,1,Fiber optic,0,1,1,0,0,0,Month-to-month,1,Bank transfer (automatic),84.8,371.9,1 +Female,1,0,0,21,0,0 phone service,DSL,1,0,0,0,1,0,Month-to-month,1,Electronic check,41.9,840.1,1 +Female,0,0,0,10,1,0,Fiber optic,0,0,0,0,0,1,Month-to-month,1,Bank transfer (automatic),80.25,846,1 +Female,1,0,0,16,1,1,DSL,1,0,0,0,0,0,Month-to-month,1,Electronic check,54.1,889,0 +Male,0,1,0,64,1,1,Fiber optic,0,1,0,1,1,1,Two year,1,Electronic check,105.25,6823.4,0 +Male,1,1,0,27,0,0 phone service,DSL,1,0,0,0,0,0,Month-to-month,0,Credit card (automatic),30.75,805.1,1 +Male,0,1,1,42,1,1,Fiber optic,1,0,0,0,1,1,One year,0,Electronic check,97.1,4016.75,0 +Male,0,0,1,5,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,20.2,83.75,0 +Female,0,0,0,41,1,0,Fiber optic,0,1,0,1,1,1,Two year,1,Bank transfer (automatic),98.8,3959.15,0 +Male,0,1,0,58,1,0,DSL,0,0,1,0,0,0,One year,0,Credit card (automatic),50.3,2878.55,0 +Female,0,0,0,47,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,20.55,945.7,0 +Male,0,0,0,18,1,0,Fiber optic,0,0,0,1,0,0,Month-to-month,1,Bank transfer (automatic),75.9,1373.05,0 +Male,0,0,0,5,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,0,Bank transfer (automatic),96.5,492.55,1 +Male,0,1,0,23,1,0,DSL,1,0,0,0,0,1,Month-to-month,1,Bank transfer (automatic),59.95,1406,0 +Female,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.15,19.15,0 +Male,0,1,0,71,1,1,Fiber optic,0,0,1,0,1,1,One year,1,Credit card (automatic),98.65,6962.85,0 +Male,0,1,1,72,1,1,Fiber optic,1,1,1,1,1,1,Two year,1,Credit card (automatic),112.6,8126.65,0 +Male,0,1,1,33,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.6,690.25,0 +Male,0,0,0,2,1,1,Fiber optic,0,0,0,0,1,0,Month-to-month,1,Electronic check,85.65,181.5,1 +Male,0,0,1,24,0,0 phone service,DSL,1,0,0,1,0,0,Two year,0,Mailed check,35.75,830.8,0 +Female,0,1,1,56,1,1,Fiber optic,0,1,1,1,0,1,One year,1,Credit card (automatic),99.75,5608.4,0 +Male,0,0,0,37,1,0,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Credit card (automatic),96.1,3646.8,0 +Female,0,0,0,43,1,1,DSL,0,1,1,1,1,1,One year,0,Credit card (automatic),85.1,3662.25,0 +Male,0,0,0,1,0,0 phone service,DSL,0,0,0,0,0,0,Month-to-month,0,Mailed check,25.35,25.35,0 +Female,0,1,0,25,1,1,Fiber optic,0,0,1,1,1,1,Month-to-month,1,Bank transfer (automatic),104.95,2566.5,1 +Female,0,0,0,61,1,1,DSL,1,1,1,1,1,1,Two year,0,Electronic check,89.65,5308.7,0 +Male,0,0,0,17,1,1,Fiber optic,0,0,1,1,0,0,One year,0,Electronic check,86.75,1410.25,0 +Female,0,1,0,41,1,0,Fiber optic,0,1,1,1,0,0,One year,1,Bank transfer (automatic),86.2,3339.05,0 +Male,0,0,0,1,1,0,DSL,0,1,0,0,0,0,Month-to-month,0,Mailed check,50.65,50.65,1 +Male,1,1,1,72,0,0 phone service,DSL,1,1,1,1,1,1,Two year,1,Bank transfer (automatic),64.8,4732.35,0 +Male,0,0,0,1,1,1,Fiber optic,0,1,0,0,0,1,Month-to-month,1,Credit card (automatic),90.85,90.85,1 +Male,0,0,0,48,1,1,Fiber optic,0,1,1,0,1,1,Two year,1,Bank transfer (automatic),108.1,5067.45,0 +Male,1,1,0,11,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.95,214.75,1 +Female,1,1,0,55,1,1,Fiber optic,0,1,1,0,0,0,Month-to-month,1,Electronic check,85.45,4874.7,1 +Female,0,0,0,42,0,0 phone service,DSL,1,1,1,1,0,1,One year,1,Electronic check,54.75,2348.45,0 +Male,0,0,0,44,1,1,Fiber optic,0,0,1,0,0,1,Month-to-month,1,Mailed check,90.4,4063,0 +Male,0,0,1,1,1,0,DSL,0,0,0,0,0,0,Month-to-month,0,Electronic check,44,44,0 +Female,0,0,0,27,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Bank transfer (automatic),95.6,2595.25,0 +Male,1,1,0,27,1,1,DSL,0,1,1,1,1,1,Month-to-month,1,Bank transfer (automatic),84.8,2309.55,0 +Female,1,0,0,2,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Electronic check,44.3,89.3,0 +Female,0,0,0,19,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,0,Mailed check,19.9,367.55,0 +Female,1,1,0,42,1,1,Fiber optic,0,0,0,0,1,1,Month-to-month,1,Electronic check,95.05,3944.5,0 +Female,0,0,0,66,1,1,DSL,1,1,1,1,1,1,Two year,1,Credit card (automatic),90.05,5965.95,0 +Male,1,1,0,33,1,1,Fiber optic,1,0,1,1,1,1,Month-to-month,1,Bank transfer (automatic),109.9,3694.7,0 +Female,0,0,0,34,1,0,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Credit card (automatic),73.95,2524.45,1 +Male,1,0,0,33,0,0 phone service,DSL,0,0,1,1,1,1,Month-to-month,1,Electronic check,54.6,1803.7,0 +Female,0,1,1,23,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,20.05,415.1,0 +Female,0,0,0,32,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Bank transfer (automatic),19.75,624.15,0 +Female,0,0,0,11,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Credit card (automatic),20.05,237.7,0 +Male,0,1,1,69,1,1,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Credit card (automatic),99.45,7007.6,1 +Female,0,1,0,68,1,0,DSL,1,0,0,1,0,0,One year,0,Bank transfer (automatic),55.9,3848.8,0 +Male,0,0,0,20,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,19.7,419.4,0 +Male,0,0,0,72,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Credit card (automatic),19.8,1468.75,0 +Male,0,1,1,60,1,1,Fiber optic,1,0,1,0,1,0,One year,1,Bank transfer (automatic),95.4,5812,0 +Male,1,1,1,32,1,0,Fiber optic,0,1,0,0,1,1,Month-to-month,1,Mailed check,93.95,2861.45,0 +Female,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,19.9,19.9,1 +Male,0,0,0,1,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,19.6,19.6,1 +Male,1,0,0,3,1,1,Fiber optic,0,1,0,0,0,0,Month-to-month,1,Electronic check,81.35,233.7,1 +Female,0,1,0,46,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,1,Bank transfer (automatic),24.45,1066.15,0 +Female,0,1,0,29,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Electronic check,74.95,2149.05,0 +Male,0,0,0,51,1,1,Fiber optic,0,1,1,1,0,0,Month-to-month,1,Electronic check,87.35,4473,0 +Female,1,1,0,48,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),70.65,3545.05,0 +Female,0,1,1,16,1,0,Fiber optic,0,1,0,0,0,0,Month-to-month,0,Bank transfer (automatic),73.25,1195.75,0 +Male,0,1,0,70,1,1,Fiber optic,0,1,1,1,0,1,Two year,1,Bank transfer (automatic),98.7,6858.9,0 +Female,0,1,1,40,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Mailed check,24.8,1024.7,0 +Female,0,1,1,22,1,0,Fiber optic,0,1,0,0,0,1,Month-to-month,0,Electronic check,83.3,1845.9,1 +Female,1,0,0,1,1,0,Fiber optic,0,0,1,0,0,0,Month-to-month,1,Bank transfer (automatic),75.3,75.3,1 +Female,0,1,1,5,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,24.3,132.25,0 +Female,0,0,0,7,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),69.85,515.45,0 +Male,0,0,1,29,1,1,Fiber optic,0,1,1,1,0,1,One year,0,Credit card (automatic),100.55,2830.45,0 +Male,0,1,1,44,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,25.7,1110.5,0 +Female,0,0,0,10,0,0 phone service,DSL,1,1,1,0,0,0,Month-to-month,0,Bank transfer (automatic),40.7,449.3,0 +Female,1,1,0,55,1,1,DSL,0,0,0,0,0,0,Month-to-month,0,Bank transfer (automatic),51.65,2838.55,0 +Male,1,1,0,52,1,1,Fiber optic,0,1,1,0,1,1,One year,1,Bank transfer (automatic),105.1,5376.4,0 +Female,0,1,1,10,1,1,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Electronic check,85.95,858.6,0 +Male,0,0,0,18,1,1,Fiber optic,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),75.6,1395.05,0 +Female,0,1,0,68,1,1,DSL,1,0,0,1,0,0,One year,0,Bank transfer (automatic),58.25,3975.7,0 +Female,0,1,1,61,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Month-to-month,1,Mailed check,19.4,1182.55,1 +Female,0,1,1,72,0,0 phone service,DSL,1,1,1,1,1,1,Two year,1,Credit card (automatic),65.2,4784.45,0 +Male,0,0,0,2,1,0,DSL,1,0,1,0,0,0,Month-to-month,0,Electronic check,53.45,119.5,0 +Male,0,0,0,12,1,0,DSL,0,0,0,0,0,0,Month-to-month,1,Bank transfer (automatic),45.4,518.9,1 +Male,0,1,0,41,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Electronic check,19.75,899.45,0 +Female,0,0,0,26,0,0 phone service,DSL,0,0,1,1,1,0,Month-to-month,1,Credit card (automatic),44.45,1183.8,0 +Male,0,1,0,36,1,0,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Bank transfer (automatic),20.85,720.05,0 +Male,0,1,1,72,1,1,Fiber optic,1,1,1,1,1,1,Two year,1,Credit card (automatic),114.05,8468.2,0 +Male,0,1,0,35,1,1,Fiber optic,0,0,0,1,0,1,Month-to-month,1,Bank transfer (automatic),89.85,3161.2,0 +Male,0,0,0,1,1,0,DSL,0,0,1,1,0,0,Month-to-month,0,Mailed check,55.05,55.05,0 +Female,0,1,1,16,1,1,Fiber optic,1,0,1,1,1,1,Month-to-month,1,Electronic check,112.95,1882.55,0 +Female,1,0,0,49,1,1,Fiber optic,1,0,0,0,1,1,Month-to-month,0,Credit card (automatic),101.55,5070.4,0 +Female,0,1,0,54,1,1,Fiber optic,1,1,1,1,1,1,One year,1,Mailed check,114.65,6049.5,0 +Female,0,0,1,18,1,1,DSL,1,0,0,0,0,1,Month-to-month,0,Electronic check,64.8,1166.7,0 +Female,0,1,0,36,1,0,Fiber optic,0,0,0,0,0,1,Month-to-month,0,Electronic check,80.4,2937.65,0 +Female,0,0,0,60,1,1,Fiber optic,0,1,1,0,1,1,Month-to-month,1,Electronic check,105.9,6396.45,1 +Female,0,0,0,1,1,0,Fiber optic,0,0,0,0,0,0,Month-to-month,0,Mailed check,69.55,69.55,1 +Female,0,1,1,52,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,Two year,1,Bank transfer (automatic),25.05,1270.25,0 +Male,0,1,1,8,1,0,Fiber optic,0,0,1,0,1,1,Month-to-month,1,Electronic check,94.75,759.55,0 +Male,0,1,1,72,1,0,Fiber optic,1,1,0,1,1,1,Two year,0,Bank transfer (automatic),105.5,7611.55,0 +Female,0,0,0,64,1,1,0,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,0 internet service,One year,0,Mailed check,24.7,1642.75,0 +\. + + +select xgboost_sk_classifier('telecom', 'telecom_mode', '"customerID"', '"Churn"', '"MonthlyCharges", "TotalCharges"', NULL, $${'booster': ['gbtree'], 'eta': (0.1), 'max_depth': 5, 'objective': ['binary:logistic']}$$, TRUE); +\d+ telecom_mode +select xgboost_sk_predict('telecom', 'telecom_mode', 'outtable', '"customerID"', TRUE); \ No newline at end of file diff --git a/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.py_in b/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.py_in new file mode 100644 index 0000000..00cc866 --- /dev/null +++ b/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.py_in @@ -0,0 +1,488 @@ +# -*- coding:utf-8 -*- +# Copyright (c) 2020 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------- +# Xgboost regression and Classification use sklearn API. +# ---------------------------------------------------------------------- +import ast +import collections +import itertools +import plpy + +import cPickle as pickle +import pandas as pd + +from utilities.validate_args import is_var_valid +from utilities.validate_args import table_exists +from utilities.validate_args import table_is_empty +from utilities.validate_args import columns_exist_in_table +from utilities.validate_args import get_cols +from utilities.validate_args import unquote_ident +from utilities.validate_args import quote_ident +from utilities.utilities import _assert +from utilities.control import MinWarning + +from xgboost_utilities import get_features_to_use + + +def xgboost_sk_classifier(schema_madlib, input_table, model_table, id_column, dependent_varname, + str_of_features, str_of_features_to_exclude, params_str, evaluation, verbose, **kwargs): + """ + Args: + @param input_table, -- name of input table + @param model_table, -- name of output table + @param id_column + @param dependent_varname, -- name of dependent variable Y + @param str_of_features, -- name of independent variables X, Comma separated column names to be + used as the predictors, can be '*' + to include all columns except the dependent_variable + @param str_of_features_to_exclude -- Comma separated column names to be excluded if str_of_features is '*', + Ignore 'str_of_features_to_exclude' if 'str_of_features' is not '*' + @param params_str, -- names of paraters for xgboost + @param verbose -- verbose or not + """ + with MinWarning('warning'): + _validate_args(schema_madlib, input_table, model_table, id_column, + dependent_varname, str_of_features, str_of_features_to_exclude, params_str) + + _, features_list = get_features_to_use(schema_madlib, input_table, id_column, dependent_varname, + str_of_features, str_of_features_to_exclude, verbose) + + sql = '''SELECT {schema_madlib}.xgboost_grid_search('{input_table}', '{grid_search_results_tbl}', '{id_column}', + '{dependent_varname}', ARRAY{features_list}, '{xgb_type}', $${params_str}$$, {evaluation}, {verbose}); + '''.format( + schema_madlib=schema_madlib, + input_table=input_table, + grid_search_results_tbl=model_table, + id_column=id_column, + dependent_varname=dependent_varname, + features_list=features_list, + xgb_type='C', + params_str=params_str, + evaluation=evaluation, + verbose=verbose + ) + if verbose: + plpy.info(sql) + + ret = plpy.execute(sql) + return ret[0]['xgboost_grid_search'] + + +def xgboost_sk_regressor(schema_madlib, input_table, model_table, id_column, + dependent_varname, str_of_features, str_of_features_to_exclude, params_str, evaluation, verbose, **kwargs): + """ + Args: + @param input_table, -- name of input table + @param model_table, -- name of output table + @param id_column -- id column of the table + @param dependent_varname, -- name of dependent variable Y + @param str_of_features, -- name of independent variables X + @param params_str, -- {key: [value1, valule2]} + @param verbose + """ + with MinWarning('warning'): + _validate_args(schema_madlib, input_table, model_table, id_column, + dependent_varname, str_of_features, str_of_features_to_exclude, params_str) + + _, features_list = get_features_to_use(schema_madlib, input_table, id_column, dependent_varname, + str_of_features, str_of_features_to_exclude, verbose) + + sql = '''SELECT {schema_madlib}.xgboost_grid_search('{input_table}', '{grid_search_results_tbl}', '{id_column}', + '{dependent_varname}', ARRAY{features_list}, '{xgb_type}', $${params_str}$$, {evaluation}, {verbose}); + '''.format( + schema_madlib=schema_madlib, + input_table=input_table, + grid_search_results_tbl=model_table, + id_column=id_column, + dependent_varname=dependent_varname, + features_list=features_list, + xgb_type='R', + params_str=params_str, + evaluation=evaluation, + verbose=verbose + ) + if verbose: + plpy.info(sql) + + ret = plpy.execute(sql) + return ret[0]['xgboost_grid_search'] + + +def xgboost_sk_predict(schema_madlib, input_table, model_table, output_table, id_column, verbose, **kwargs): + """ + Args: + @param input_table, -- Input: name of input table + @param model_table, -- Input: name of model table + @param output_table, -- Output: save the results. + @param id_column + @param verbose + """ + with MinWarning('warning'): + _validate_predict(schema_madlib, input_table, model_table, output_table, id_column) + + sql = '''SELECT y_type, features, model FROM {model_table}'''.format( + model_table=model_table) + result = plpy.execute(sql) + + # get the type of label. + y_type = result[0]['y_type'] + features_list = result[0]['features'] + features_list = [quote_ident(x) for x in features_list] + + # 1) Extract feature names from information_schema + all_columns = get_cols(input_table) + + for f in features_list: + if f not in all_columns: + plpy.error('''xgboost_sk_predict, {f} donot find in training table.'''.format(str(f))) + + # 2) get model + model = result[0]['model'] + model = pickle.loads(model) + + # 3) Fetch dataset for model training + sql = """select {col} from {input_table} order by {id_column}""".format( + id_column=id_column, + col=','.join(features_list), + input_table=input_table + ) + + if verbose: + plpy.info("xgboost_sk_predict() " + sql) + + result = plpy.execute(sql) + test_df = pd.DataFrame.from_records(result) + + features_list = [unquote_ident(x) for x in features_list] + features = filter(lambda x: x in test_df.columns, features_list) + if verbose: + plpy.info("xgboost_sk_predict() - features: " + str(features)) + test_df = test_df[features] + + # get id + sql = """select {id_column} from {input_table} order by {id_column}""".format( + id_column=id_column, + input_table=input_table + ) + result = plpy.execute(sql) + + id_df = pd.DataFrame.from_records(result) + id_df = id_df.reset_index(drop=True) + id_df = id_df[id_df.columns[0]] + + # start to predict + predict_res = model.predict(test_df) + + sql = '''CREATE TABLE {output_table} (id bigint, prediction {y_type})'''.format( + output_table=output_table, y_type=y_type) + if verbose: + plpy.info("xgboost_sk_predict() " + sql) + + plpy.execute(sql) + _assert(len(predict_res) == len(id_df), + "Xgboost predict error: len(id) != len(predict)") + + sql = """INSERT INTO {output_table} VALUES ({id}, {prediction});""" + for i in range(len(id_df)): + plpy.execute( + sql.format(id=id_df[i], + prediction=predict_res[i], + output_table=output_table + ) + ) + +# ------------------------------------------------------------------------------ +# -- validate args ------------------------------------------------------------ +# ------------------------------------------------------------------------------ +def _validate_args(schema_madlib, input_table, model_table, id_column, + dependent_varname, str_of_features, str_of_features_to_exclude, params_str): + """ + @brief validate the arguments + """ + # 1, check input_table: name. + _assert(input_table and input_table.strip().lower() not in ('null', ''), + "Xgboost error: Invalid data table name!") + _assert(table_exists(input_table), + "Xgboost error: Data table does not exist!") + _assert(not table_is_empty(input_table), + "Xgboost error: Data table is empty!") + + # 2, check input table: id, Dependent variable, features. + _assert(columns_exist_in_table(input_table, [id_column]), + "Xgboost error: id_column doesnot exist in the input_table") + + _assert(dependent_varname is None or dependent_varname.strip().lower() not in ('null', ''), + "Xgboost error: Invalid dependent column name!") + + _assert(is_var_valid(input_table, dependent_varname), + "Xgboost error: Invalid dependent variable ({0}).".format(dependent_varname)) + + _assert(str_of_features and str_of_features.strip(), + "Xgboost error: Features to include is empty.") + + if str_of_features.strip() != '*': + _assert(is_var_valid(input_table, str_of_features), + "Xgboost error: Invalid feature list ({0})".format(str_of_features)) + + if str_of_features.strip() != '*': + _assert(dependent_varname not in str_of_features, + "Xgboost error: Dependent_varname in str_of_features") + + # 3, check output table. + _assert(model_table is not None and model_table.strip().lower() not in ('null', ''), + "Xgboost error: Invalid output table name!") + _assert(not table_exists(model_table, only_first_schema=True), + "Output table name already exists. Drop the table before calling the function.") + + # 4, check params_str + if params_str != '': + # let xgboost to check the parameter + pass + + +def _validate_predict(schema_madlib, input_table, model_table, output_table, id_column): + # TODO, check all tables. + _assert(input_table is not None and input_table.strip().lower() + not in ('null', ''), "Xgboost error: Invalid data table name!") + _assert(table_exists(input_table), + "Xgboost error: Data table does not exist!") + _assert(not table_is_empty(input_table), + "Xgboost error: Data table is empty!") + + _assert(model_table is not None and model_table.strip().lower() + not in ('null', ''), "Xgboost error: Invalid Model table name!") + _assert(table_exists(model_table), + "Xgboost error: Model table does not exist!") + _assert(not table_is_empty(model_table), + "Xgboost error: Model table is empty!") + _assert(output_table is not None and output_table.strip().lower() + not in ('null', ''), "Xgboost error: Invalid output table name!") + _assert(not table_exists(output_table, only_first_schema=True), + "Output table name already exists. Drop the table before calling the function.") + +# ------------------------------------------------------------------------------ +# -- Online help function ------------------------------------------------------ +# ------------------------------------------------------------------------------ +def xgboost_sk_classifier_help_message(schema_madlib, message, **kwargs): + """ Help message for Xgboost XGBClassifier sklearn API + + @brief + Args: + @param schema_madlib string, Name of the schema madlib + @param message string, Help message indicator + + Returns: + String. Contain the help message string + """ + if not message: + help_string=""" + ----------------------------------------------------------------------- + SUMMARY + ----------------------------------------------------------------------- + XGBoost sklearn API Training Library. + + For more details on function usage: + SELECT {schema_madlib}.xgboost_sk_classifier('usage') + """ + elif message in ['usage', 'help', '?']: + help_string=""" + ----------------------------------------------------------------------- + USAGE + ----------------------------------------------------------------------- + SELECT {schema_madlib}.xgboost_sk_classifier( + input_table TEXT, -- in, Data table name + model_table TEXT, -- out, save model and features + id_column TEXT, -- id column of the dataset table + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, Comma separated column names to be + used as the predictors, can be '*' + to include all columns except the + dependent_variable + str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to + exlude from the predictors list, + if str_of_features is '*' + params_str TEXT, -- xgboost parameters + evaluation BOOL, -- evaluation the model? default FALSE + verbose bool -- Boolean, whether to print more info, default is False + ); + + ----------------------------------------------------------------------- + parameters + ----------------------------------------------------------------------- + below is optional parameters: + + evalution: + default is FALSE, if you set evaluation=TRUE, it will split the training dataset(0.8 for training, 0.2 for testing), + and calculate metrics. + For binary classification: ['precision', 'recall', 'fscore', 'support'] + OR for multi-calssification: ['kappa', 'acc'] + + params_str: + For example, if the input is: + $${{'booster': ['gbtree'], 'eta': 0.1, 'max_depth': [5,6,7], 'objective': ['binary:logistic']}}$$ + OR + $${{'booster': ('gbtree',), 'eta': 0.1, 'max_depth': (5,6,7), 'objective': ('binary:logistic',)}}$$ + Then, cartesian_product is: + [('booster = gbtree', 'eta = 0.1', 'max_depth = 5', 'objective = binary:logistic'), + ('booster = gbtree', 'eta = 0.1', 'max_depth = 6', 'objective = binary:logistic'), + ('booster = gbtree', 'eta = 0.1', 'max_depth = 7', 'objective = binary:logistic')] + + select {schema_madlib}.xgboost_sk_Classifier('inputTable', 'output_modelTable', 'id', 'Y', 'x1, x2, x3', NULL, + $${{'booster': ['gbtree'], 'eta': 0.1, 'max_depth': 5, 'objective': ('binary:logistic',)}}$$, '', TRUE, TRUE); + + ----------------------------------------------------------------------- + OUTPUT + ----------------------------------------------------------------------- + The output table ('model_table' above) has the following columns: + id -- index + train_timestamp -- training time + model_name -- model name + y_type -- the type of independent variable + metrics -- default is OFF, evaluation, for binary classification: ['precision', 'recall', 'fscore', 'support'] + OR for multi-calssification: ['kappa', 'acc'] + features -- feature in binary model + model -- binary model + params -- xgboost prarms, {{key:value}} + """ + else: + help_string="No such option. Use {schema_madlib}.xgboost_sk_classifier()" + + return help_string.format(schema_madlib=schema_madlib) + + +def xgboost_sk_regressor_help_message(schema_madlib, message, **kwargs): + """ Help message for Xgboost XGBRegressor sklearn API + + @brief + Args: + @param schema_madlib string, Name of the schema madlib + @param message string, Help message indicator + + Returns: + String. Contain the help message string + """ + if not message: + help_string=""" + ----------------------------------------------------------------------- + SUMMARY + ----------------------------------------------------------------------- + XGBoost sklearn API Training Library. + + For more details on function usage: + SELECT {schema_madlib}.xgboost_sk_regressor('usage') + """ + elif message in ['usage', 'help', '?']: + help_string=""" + ----------------------------------------------------------------------- + USAGE + ----------------------------------------------------------------------- + SELECT {schema_madlib}.xgboost_sk_regressor( + input_table TEXT, -- in, dataset table + model_table TEXT, -- out, save model and features + id_column TEXT, -- id column of the dataset table + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, Comma separated column names to be + used as the predictors, can be '*' + to include all columns except the + dependent_variable + str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to + exlude from the predictors list, if str_of_features is '*' + params_str TEXT, -- xgboost parameters + evaluation BOOL, -- evaluation the model? default FALSE + verbose BOOL -- Boolean, whether to print more info, default is False + ); + + ----------------------------------------------------------------------- + parameters + ----------------------------------------------------------------------- + below is optional parameters: + + evalution: + default is FALSE, if you set evaluation=TRUE, it will split the training dataset(0.8 for training, 0.2 for testing), + and calculate metrics: ['mae', 'mse', 'R2squared', 'rmse'] + + params_str: + For example, if the input is: + $${{'booster': ['gbtree'], 'eta': 0.1, 'max_depth': [5,6,7], 'objective':['reg:linear']}}$$ + OR + $${{'booster': ('gbtree',), 'eta': 0.1, 'max_depth': (5,6,7), 'objective':('reg:linear')}}$$ + Then, cartesian_product is: + [('booster = gbtree', 'eta = 0.1', 'max_depth = 5', 'objective = reg:linear'), + ('booster = gbtree', 'eta = 0.1', 'max_depth = 6', 'objective = reg:linear'), + ('booster = gbtree', 'eta = 0.1', 'max_depth = 7', 'objective = reg:linear')] + + select {schema_madlib}.xgboost_sk_Regressor('input_TABLE', 'output_model_table', 'id', 'Y', 'x1, x2, x3, x4', 'x5, x6', + $${{'booster': ['gbtree'], 'eta': (0.1), 'max_depth': (5,6,7), 'objective':['reg:linear']}}$$, TRUE, TRUE); + + ----------------------------------------------------------------------- + OUTPUT + ----------------------------------------------------------------------- + The output table ('model_table' above) has the following columns: + train_timestamp -- training time + model_name -- model name + y_type -- the type of independent variable + metrics -- if evaluation is TRUE, you will get ['mae', 'mse', 'R2squared', 'rmse'] + features -- feature in binary model + model -- binary model + params -- xgboost prarms, {{key:value}} + params_indx -- index + """ + else: + help_string="No such option. Use {schema_madlib}.xgboost_sk_regressor()" + + return help_string.format(schema_madlib=schema_madlib) + + +def xgboost_sk_predict_help_message(schema_madlib, message, **kwargs): + """ Help message for Xgboost predict sklearn API + + @brief + Args: + @param schema_madlib string, Name of the schema madlib + @param message string, Help message indicator + + Returns: + String. Contain the help message string + """ + if not message: + help_string=""" + ----------------------------------------------------------------------- + SUMMARY + ----------------------------------------------------------------------- + XGBoost sklearn API Training Library. + + For more details on function usage: + SELECT {schema_madlib}.xgboost_sk_predict('usage') + """ + elif message in ['usage', 'help', '?']: + help_string=""" + ----------------------------------------------------------------------- + USAGE + ----------------------------------------------------------------------- + SELECT {schema_madlib}.xgboost_sk_predict( + input_table TEXT, -- in, test dataset + model_table TEXT, -- in, model table from training + output_table TEXT, -- out, prediction of the test + id_column TEXT, -- id column of the test dataset + verbose BOOL -- Boolean, whether to print more info, default is False + ); + """ + else: + help_string="No such option. Use {schema_madlib}.xgboost_sk_predict()" + + return help_string.format(schema_madlib=schema_madlib) diff --git a/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.sql_in b/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.sql_in new file mode 100644 index 0000000..19df7d4 --- /dev/null +++ b/src/ports/postgres/modules/xgboost_gs/xgboost_sklearn.sql_in @@ -0,0 +1,197 @@ +/* + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + */ + +m4_include(`SQLCommon.m4') + +---------------------------------------------------------------------------------------------- +------ Help messages +---------------------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier( + message TEXT +) RETURNS TEXT AS $XX$ + PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_classifier_help_message) +$XX$ LANGUAGE plpythonu IMMUTABLE; +------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier() +RETURNS TEXT AS $XX$ + SELECT MADLIB_SCHEMA.xgboost_sk_classifier(''::TEXT); +$XX$ +LANGUAGE sql IMMUTABLE; + +----------------------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor( + message TEXT +) RETURNS TEXT AS $XX$ + PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_regressor_help_message) +$XX$ LANGUAGE plpythonu IMMUTABLE; +------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor() +RETURNS TEXT +AS $XX$ + SELECT MADLIB_SCHEMA.xgboost_sk_regressor(''::TEXT); +$XX$ +LANGUAGE sql IMMUTABLE; +---------------------------------------------------------------------------------------------- + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_predict( + message TEXT +) RETURNS TEXT AS $$ + PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_predict_help_message) +$$ LANGUAGE plpythonu IMMUTABLE; +------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_predict() +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.xgboost_sk_predict(''::TEXT); +$$ +LANGUAGE sql IMMUTABLE; +------------------------------------------------------------- + + +---------------------------------------------------------------------------------------------- +------ xgboost Classifier Algo +---------------------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier( + input_table TEXT, -- in, dataset table + model_table TEXT, -- out, save model and features + id_column TEXT, -- id column of the dataset table + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, + str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list. + params_str TEXT, -- xgboost parameters + evaluation BOOL, -- evaluation the training model, default FALSE. + verbose BOOL +) +RETURNS TEXT AS $$ +PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_classifier) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier( + input_table TEXT, + model_table TEXT, + id_column TEXT, + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, + str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list. + params_str TEXT, + evaluation BOOL -- evaluation the training model, default FALSE. +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.xgboost_sk_classifier($1, $2, $3, $4, $5, $6, $7, $8, FALSE); +$$ LANGUAGE sql; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier( + input_table TEXT, + model_table TEXT, + id_column TEXT, + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, + str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list. + params_str TEXT +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.xgboost_sk_classifier($1, $2, $3, $4, $5, $6, $7, FALSE); +$$ LANGUAGE sql; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier( + input_table TEXT, + model_table TEXT, + id_column TEXT, + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, + str_of_features_to_exclude TEXT -- List of column names (comma-separated string) to exlude from the predictors list. +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.xgboost_sk_classifier($1, $2, $3, $4, $5, $6, ''); +$$ LANGUAGE sql; + +---------------------------------------------------------------------------------------------- +------ xgboost XGBRegressor +---------------------------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor( + input_table TEXT, -- in, dataset table + model_table TEXT, -- out, save model and features + id_column TEXT, -- id column of the dataset table + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, + str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list. + params_str TEXT, -- xgboost parameters + evaluation BOOL, -- evaluation the training model, default FALSE. + verbose BOOL +) +RETURNS TEXT AS $$ +PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_regressor) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor( + input_table TEXT, + model_table TEXT, + id_column TEXT, + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, + str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list. + params_str TEXT, + evaluation BOOL -- evaluation the training model, default FALSE. +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.xgboost_sk_regressor($1, $2, $3, $4, $5, $6, $7, $8, FALSE); +$$ LANGUAGE sql; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor( + input_table TEXT, + model_table TEXT, + id_column TEXT, + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, + str_of_features_to_exclude TEXT, -- List of column names (comma-separated string) to exlude from the predictors list. + params_str TEXT +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.xgboost_sk_regressor($1, $2, $3, $4, $5, $6, $7, FALSE); +$$ LANGUAGE sql; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor( + input_table TEXT, + model_table TEXT, + id_column TEXT, + dependent_varname TEXT, -- y, class label + str_of_features TEXT, -- X, + str_of_features_to_exclude TEXT -- List of column names (comma-separated string) to exlude from the predictors list. +) +RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.xgboost_sk_regressor($1, $2, $3, $4, $5, $6, ''); +$$ LANGUAGE sql; + +------------------------------------------------ +-- XGBoost predict +------------------------------------------------ +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_predict( + input_table TEXT, -- in, test dataset + model_table TEXT, -- in, model table from training + output_table TEXT, -- out, prediction of the test + id_column TEXT, -- id column of the test dataset + verbose BOOL +) RETURNS TEXT AS $$ +PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_predict) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_predict( + input_table TEXT, + model_table TEXT, + output_table TEXT, + id_column TEXT +) RETURNS TEXT AS $$ + SELECT MADLIB_SCHEMA.xgboost_sk_predict($1, $2, $3, $4, FALSE); +$$ LANGUAGE sql; diff --git a/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.py_in b/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.py_in new file mode 100644 index 0000000..2355ba7 --- /dev/null +++ b/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.py_in @@ -0,0 +1,420 @@ +# -*- coding:utf-8 -*- +# Copyright (c) 2020 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms +# and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, +# WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------- +# Xgboost regression and Classification use sklearn API. +# ---------------------------------------------------------------------- +import ast +import collections +import itertools +import plpy +import re + +import xgboost as xgb +import numpy as np +import cPickle as pickle +import pandas as pd + +from sklearn.model_selection import train_test_split +from sklearn.metrics import precision_recall_fscore_support +from sklearn.metrics import mean_squared_error +from sklearn.metrics import mean_absolute_error +from sklearn.metrics import r2_score +from sklearn.metrics import cohen_kappa_score +from sklearn.metrics import accuracy_score + +from utilities.utilities import unique_string +from utilities.utilities import split_quoted_delimited_str +from utilities.utilities import _assert +from utilities.control import MinWarning +from utilities.validate_args import get_cols +from utilities.validate_args import get_expr_type +from utilities.validate_args import quote_ident +from utilities.validate_args import unquote_ident +from utilities.validate_args import _get_table_schema_names + +# xgboost donot support category, xgboost only deals with numeric columns +support_type_list = ('TINYINT', 'SMALLINT', 'INTEGER', 'BINARY_INTEGER', 'BIGINT', 'DECIMAL', + 'SMALLSERIAL', 'SERIAL', 'BIGSERIAL', + 'REAL', 'FLOAT4', 'DOUBLE PRECISION', 'FLOAT8', 'FLOAT', 'BINARY_DOUBLE', 'DEC', + 'BOOLEAN') + + +def get_features_to_use(schema_madlib, input_table, id_column, dependent_varname, str_of_features, str_of_features_to_exclude, verbose=False): + """ + get all columns and used columns. + return: quote_idented column. + """ + other_col_set = set([id_column, dependent_varname]) + other_col_set |= set(unquote_ident(i) for i in [id_column, dependent_varname]) + + if str_of_features.strip() == '*': + all_col = get_cols(input_table) # get all columns in order with quote_ident! + all_col_set = set(all_col) + exclude_set = set(split_quoted_delimited_str(str_of_features_to_exclude)) + feature_set = all_col_set - exclude_set + filtered_feature_list = list(feature_set - other_col_set) + filtered_feature_list = [t for t in all_col if t in filtered_feature_list] # columns order is important + else: + all_col_set = set(get_cols(input_table)) + feature_list = split_quoted_delimited_str(str_of_features) + feature_exclude = split_quoted_delimited_str(str_of_features_to_exclude) + return_set = set(feature_list) - set(feature_exclude) - other_col_set + # instead of returning list(return_set) we create a list that has + # elements in same order as original feature_list + filtered_feature_list = [feat for feat in feature_list if feat in return_set] + + if verbose: + plpy.info("get_features_to_use() " + str(all_col_set) + " -- " + str(filtered_feature_list)) + return all_col_set, filtered_feature_list + +def get_column_and_ret_type(schema_madlib, input_table, id_column, dependent_varname, features_list, verbose=False): + ''' + 1. check the data type of the input table. + 2. get the type of dependent_varname. + ''' + # Extract feature names from information_schema, and check type. + col_list = features_list + [dependent_varname] # X + [y] with unquote_idented + + # input_table = . + s_name, t_name = _get_table_schema_names(input_table, True) + sql = """SELECT + column_name, data_type + FROM + information_schema.columns + WHERE + table_name = '{t_name}' AND table_schema IN {s_name} AND column_name IN {col_list}; + """.format( + t_name=t_name, + s_name=s_name, + col_list=str(col_list).replace('[', '(').replace(']', ')').replace('"', '') + ) + if verbose: + plpy.info("get_column_and_ret_type() " + sql) + + result = plpy.execute(sql) + + # xgboost do not support category, xgboost only deals with numeric columns + for r in result: + if verbose: + plpy.info("get_column_and_ret_type() check type: " + r['column_name'] + " --> " + r['data_type']) + if r['data_type'].upper() not in support_type_list: + plpy.error("Wrong Xgboost type, " + r['column_name'] + " is " + r['data_type']) + + # save the type of dependent_variable. + y_type = get_expr_type(dependent_varname, input_table) + + if verbose: + plpy.info("get_column_and_ret_type() lable type is: " + str(y_type)) + + return y_type + + +def save_table_as_bytea(schema_madlib, input_table, id_column, + dependent_varname, features_list, verbose, **kwargs): + """ + transfrom table => dataframe => dumpfile + """ + # 1) Extract feature names from information_schema + col_list = features_list + [dependent_varname] # X + [y], quote_idented + + # 2) Fetch dataset for model training + sql = """select {id_column}, {col} from {input_table}""".format( + id_column=id_column, + col=','.join(col_list), + input_table=input_table + ) + + if verbose: + plpy.info("save_table_as_bytea() " + str(sql)) + + result = plpy.execute(sql) + # => df + df = pd.DataFrame.from_records(result) + + if verbose: + plpy.info("save_table_as_bytea() " + str(df.columns)) + + # dump dataframe. + return pickle.dumps(df) + + +def xgboost_train_parallel(schema_madlib, dframe, dependent_varname, features_list, + xgb_type, params, evaluation, verbose, **kwargs): + """ + xgboost grid search Algorithm + Args: + @param dframe, -- name of input table + @param dependent_varname, -- name of dependent variable Y + @param features_list, -- name of independent variables X + @param xgb_type, -- 'C' or 'R', classificaiton or Regression + @param params, -- dictionary. params = "{'eta':0.1, 'max_depth':2, 'min_child_weight':3}" + """ + # 1) Load the dataset for model training + df = pickle.loads(dframe) + + # 2) Train XGBoost model & return a serialized representation to store in table, keep the order. + features_list = [unquote_ident(x) for x in features_list] + features = filter(lambda x: x in df.columns, features_list) + if verbose: + plpy.info("xgboost_train_parallel(): " + str(df.columns) + " <>" + str(features)) + X = df[features] + y = df[unquote_ident(dependent_varname)] + + if evaluation: + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) + else: + X_train, y_train = X, y + + if verbose: + plpy.info("xgboost_train_parallel() X_train.columns: " + str(X_train.columns)) + plpy.info("xgboost_train_parallel() X_train.dtypes: " + str(X_train.dtypes)) + + _assert(xgb_type in ('c', 'C', 'R', 'r'), + "Xgboost error: xgb_type must be 'C' or 'R'") + + # Train gradient boosted trees + # tuple string => dictionary + if type(params) == str: + params = ast.literal_eval(params) + if type(params) != tuple: + plpy.error("Wrong XGboost parameters " + str(params)) + + params_dict = dict() + for p in params: + k, v = p.split("=") + params_dict[k.strip()] = v.strip() + + if verbose: + plpy.info("xgboost_train_parallel() params list: " + str(params_dict)) + plpy.info("xgboost_train_parallel() train type: " + xgb_type) + + model_metrics = "evalution is OFF, no metrics" + + if xgb_type == 'C' or xgb_type == 'c': + categories = len(set(y_train)) + plpy.info("xgboost_train_parallel() Classification categories: " + str(categories)) + + if categories < 2: + plpy.error("Wrong Xgboost Categories: " + str(categories)) + + gbm = xgb.XGBClassifier(**params_dict) + + # Fit model + gbm.fit(X_train, y_train) + + # 3) Compute and return model metrics score + if evaluation: + y_pred_test = gbm.predict(X_test) + + if categories == 2: + scores = np.array(precision_recall_fscore_support(y_test, y_pred_test)).transpose() + + # precision: TP / (TP + FP) where tp is the number of true positives and fp the number of false positives. + # recall: TP / (TP + FN) where tp is the number of true positives and fn the number of false negatives. + # fscore: F1=2PR/(P+R) + # support: is the number of occurrences of each class in y_true + model_metrics = pd.DataFrame(scores, columns=['precision', 'recall', 'f1score', 'support']) + model_metrics['class'] = gbm.classes_ + model_metrics = model_metrics[['class'] + ['precision', 'recall', 'f1score', 'support']] + + model_metrics = model_metrics.to_string(index=False) + + else: + model_metrics = dict() + model_metrics['kappa'] = cohen_kappa_score(y_test, y_pred_test) + model_metrics['acc'] = accuracy_score(y_test, y_pred_test) + model_metrics = str(model_metrics) + + # 1) save model_metric, 2) [x], 3) dump model, 4) current parameters + return model_metrics, features, pickle.dumps(gbm), str(params) + + else: + gbm = xgb.XGBRegressor(**params_dict) + gbm.fit(X_train, y_train) + + if evaluation: + # R2squared: coefficient of determination + metric_labels = ['mae', 'mse', 'R2squared', 'rmse'] + + y_test_pred = gbm.predict(X_test) + mae2 = mean_absolute_error(y_test_pred, y_test) + mse2 = mean_squared_error(y_test_pred, y_test) + r2 = r2_score(y_test_pred, y_test) + rmse2 = np.sqrt(mse2) + + model_metrics = {'mae': mae2, 'mse': mse2, 'r2squared': r2, 'rmse': rmse2} + model_metrics = pd.DataFrame(model_metrics, index=list('1')) + model_metrics = model_metrics.to_string() + + # 1) save model_metric, 2) [x], 3) dump model, 4) current parameters + return (model_metrics, features, pickle.dumps(gbm), params) + + +def cartesian_product(params): + """ + Expand a dict of parameters into a Cartesian product + For example, if the input is: + "{'booster': ['gbtree'], 'eta': 0.1, 'max_depth': (5,6,7), 'objective': ('binary:logistic',)}" + Then, cartesian_product is: + [('booster = gbtree', 'eta = 0.1', 'max_depth = 5', 'objective = binary:logistic'), + ('booster = gbtree', 'eta = 0.1', 'max_depth = 6', 'objective = binary:logistic'), + ('booster = gbtree', 'eta = 0.1', 'max_depth = 7', 'objective = binary:logistic')] + """ + # transfer params => Cartesian product + cp_list = [] + + for key, val in params.items(): + if val and isinstance(val, collections.Iterable): + r = [] + for v in val: + r.append("""{k} = {v}""".format(k=key, v=v)) + else: + r = ["""{k} = {v}""".format(k=key, v=val)] + cp_list.append(r) + cartesian_product = [i for i in itertools.product(*cp_list)] + return cartesian_product + + +def xgboost_grid_search(schema_madlib, input_table, grid_search_results_tbl, id_column, + dependent_varname, features_list, xgb_type, params_str, evaluation, verbose, **kwargs): + """ + xgboost grid search main entry. + """ + + y_type = get_column_and_ret_type(schema_madlib, input_table, id_column, dependent_varname, features_list, verbose) + + # Expand the grid-search parameters, Expand the params to run-grid search + if params_str is None or params_str is "" or ":" not in params_str: + params_str = "{}" + plpy.info("xgboost_grid_search() params_str type is: " + params_str) + + # str => dict + params = ast.literal_eval(re.sub("[\\t]", "", params_str).strip()) + + # trans params => Cartesian product + params_grid = cartesian_product(params) + + if verbose: + plpy.info("xgboost_grid_search() Cartesian product DONE: " + str(params_grid)) + + # Save each parameter list in the grid as a row in a distributed table + grid_search_params_temp_tbl = unique_string(desp='gridSearchXGB') + sql = """ + DROP TABLE IF EXISTS {grid_search_params_temp_tbl}; + CREATE TEMP TABLE {grid_search_params_temp_tbl} (id INT, params TEXT) + m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (id)') + m4_ifdef(`__GSDBMPP__', `DISTRIBUTE BY HASH(id)', `'); + """.format(grid_search_params_temp_tbl=grid_search_params_temp_tbl) + + if verbose: + plpy.info(sql) + + plpy.execute(sql) + + # insert grid search parameters into the g_s_p table + sql = """INSERT INTO {grid_search_params_temp_tbl} VALUES ({id}, $${val}$$);""" + + for indx, val in enumerate(params_grid): + plpy.execute( + sql.format( + grid_search_params_temp_tbl=grid_search_params_temp_tbl, + id=indx + 1, + val=val + ) + ) + + # Extract features from table and persist as serialized dataframe + col_list = features_list + [dependent_varname] # X + [y], quote_idented + sql = """ + DROP TABLE IF EXISTS {grid_search_params_temp_tbl}_df; + CREATE TEMP TABLE {grid_search_params_temp_tbl}_df m4_ifdef(`__GSDBMPP__', `DISTRIBUTE BY HASH(id)', `') + AS + ( + SELECT + 1 as id, + {schema_madlib}.__save_table_as_bytea__( + '{input_table}', + '{id_column}', + '{class_label}', + ARRAY[{features_list}], + {verbose} + ) as df + ) m4_ifdef(`__POSTGRESQL__', `', `DISTRIBUTED BY (id)'); + """.format( + schema_madlib=schema_madlib, + grid_search_params_temp_tbl=grid_search_params_temp_tbl, + input_table=input_table, + id_column=id_column, + class_label=dependent_varname, + features_list=str(features_list).replace('[', '').replace(']', ''), + verbose=verbose + ) + + if verbose: + plpy.info(sql) + + plpy.execute(sql) + + # 5) Invoke XGBoost's train by passing each row from parameter list table. This will run in parallel. + sql = """ + CREATE TABLE {grid_search_results_tbl} m4_ifdef(`__GSDBMPP__', `DISTRIBUTE BY HASH(id)', `') + AS ( + SELECT + id, + now() as train_timestamp, + '{input_table}' as source_table, + '{y_type}'::TEXT as y_type, + (mdl_results).metrics, + (mdl_results).features, + (mdl_results).model, + (mdl_results).params + FROM ( + SELECT + t1.id, + {schema_madlib}.__xgboost_train_parallel__( + df, + '{class_label}', + ARRAY[{features}], + '{xgb_type}', + params, + {evaluation}, + {verbose} + ) AS mdl_results + FROM + {grid_search_params_temp_tbl} t1, + {grid_search_params_temp_tbl}_df + )q + ); + """.format( + schema_madlib=schema_madlib, + y_type=y_type, + grid_search_results_tbl=grid_search_results_tbl, + grid_search_params_temp_tbl=grid_search_params_temp_tbl, + xgb_type=xgb_type, + features=str(features_list).replace('[', '').replace(']', '').replace(',', ','), + input_table=input_table, + class_label=dependent_varname, + evaluation=evaluation, + verbose=verbose + ) + + if verbose: + plpy.info("xgboost_grid_search() " + sql) + + plpy.execute(sql) + return """Grid search results saved in {tbl}""".format(tbl=grid_search_results_tbl) diff --git a/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.sql_in b/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.sql_in new file mode 100644 index 0000000..ea71952 --- /dev/null +++ b/src/ports/postgres/modules/xgboost_gs/xgboost_utilities.sql_in @@ -0,0 +1,81 @@ +/* + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + */ + +m4_include(`SQLCommon.m4') + +---------------------------------------------------------------------------------------------- +------ xgboost grid search Algorithm +---------------------------------------------------------------------------------------------- + +------------------------------------------------ +-- Prepare Dataset BY pandas +------------------------------------------------ +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__save_table_as_bytea__( + input_table TEXT, /* dataset table */ + id_column TEXT, /* id column */ + dependent_varname TEXT, /* class label */ + features_list text[], /* independent_v */ + verbose BOOL +) +RETURNS bytea AS $$ +PythonFunction(xgboost_gs, xgboost_utilities, save_table_as_bytea) +$$ LANGUAGE plpythonu; + + +------------------------------------------------ +-- Type for internal train results +------------------------------------------------ +DROP TYPE IF EXISTS MADLIB_SCHEMA.xgb_train_results_type cascade; +CREATE TYPE MADLIB_SCHEMA.xgb_train_results_type AS( + metrics TEXT, /* save metrices info, e.g. mae, mse */ + features TEXT[], /* save X */ + model bytea, /* dump model */ + params TEXT /* parameters used */ +); + +------------------------------------------------ +-- UDF to train XGBoost +------------------------------------------------ +CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__xgboost_train_parallel__( + dframe bytea, /* df from pandas */ + dependent_varname TEXT, /* class label */ + features_list text[], /* independent_v */ + xgb_type CHAR, /* Classifier or Regression */ + params TEXT, /* special param */ + evaluation BOOL, /* evaluation the model? default FALSE */ + verbose BOOL +) +RETURNS MADLIB_SCHEMA.xgb_train_results_type AS $$ +PythonFunction(xgboost_gs, xgboost_utilities, xgboost_train_parallel) +$$ LANGUAGE plpythonu; + +------------------------------------------------ +-- XGBoost grid search +------------------------------------------------ + +CREATE OR REPLACE function MADLIB_SCHEMA.xgboost_grid_search( + input_table TEXT, + grid_search_results_tbl TEXT, + id_column TEXT, + dependent_varname TEXT, + features_list TEXT[], + xgb_type CHAR, /* Classifier or Regressor */ + params_str TEXT, + evaluation BOOL, + verbose BOOL +)RETURNS TEXT AS $$ +PythonFunction(xgboost_gs, xgboost_utilities, xgboost_grid_search) +$$ LANGUAGE plpythonu;