From a12d19591efc3e977b54288b79498b77ccc5c2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 12 May 2017 16:47:20 +0300 Subject: [PATCH] MXS-1216: Store field real type and length in Avro schema The avro schema allows custom properties to be defined for the schema fields. The avrorouter stored extra information about the table into the schema for later use. Currently, this information is only generated by the avrorouter itself. Further improvements to the schema generator scripts need to be done. --- server/modules/routing/avro/avro_schema.c | 34 ++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/server/modules/routing/avro/avro_schema.c b/server/modules/routing/avro/avro_schema.c index 468c37c6a..444b571a4 100644 --- a/server/modules/routing/avro/avro_schema.c +++ b/server/modules/routing/avro/avro_schema.c @@ -126,9 +126,11 @@ char* json_new_schema_from_table(TABLE_MAP *map) for (uint64_t i = 0; i < map->columns; i++) { - json_array_append(array, json_pack_ex(&err, 0, "{s:s, s:s}", "name", - create->column_names[i], "type", - column_type_to_avro_type(map->column_types[i]))); + json_array_append(array, json_pack_ex(&err, 0, "{s:s, s:s, s:s, s:i}", + "name", create->column_names[i], + "type", column_type_to_avro_type(map->column_types[i]), + "real_type", create->column_types[i], + "length", create->column_lengths[i])); } json_object_set_new(schema, "fields", array); char* rval = json_dumps(schema, JSON_PRESERVE_ORDER); @@ -174,8 +176,10 @@ bool json_extract_field_names(const char* filename, TABLE_CREATE *table) { int array_size = json_array_size(arr); table->column_names = (char**)malloc(sizeof(char*) * (array_size)); + table->column_types = (char**)malloc(sizeof(char*) * (array_size)); + table->column_lengths = (int*)malloc(sizeof(int) * (array_size)); - if (table->column_names) + if (table->column_names && table->column_types && table->column_lengths) { int columns = 0; rval = true; @@ -186,6 +190,28 @@ bool json_extract_field_names(const char* filename, TABLE_CREATE *table) if (json_is_object(val)) { + json_t* value; + + if ((value = json_object_get(val, "real_type")) && json_is_string(value)) + { + table->column_types[columns] = strdup(json_string_value(value)); + } + else + { + table->column_types[columns] = strdup("unknown"); + MXS_WARNING("No \"real_type\" value defined. Treating as unknown type field."); + } + + if ((value = json_object_get(val, "length")) && json_is_integer(value)) + { + table->column_lengths[columns] = json_integer_value(value); + } + else + { + table->column_lengths[columns] = -1; + MXS_WARNING("No \"length\" value defined. Treating as default length field."); + } + json_t *name = json_object_get(val, "name"); if (name && json_is_string(name)) {