158 lines
4.2 KiB
C
158 lines
4.2 KiB
C
#ifndef _MAXAVRO_H
|
|
#define _MAXAVRO_H
|
|
/*
|
|
* Copyright (c) 2016 MariaDB Corporation Ab
|
|
*
|
|
* Use of this software is governed by the Business Source License included
|
|
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
|
|
*
|
|
* Change Date: 2020-01-01
|
|
*
|
|
* On the date above, in accordance with the Business Source License, use
|
|
* of this software will be governed by version 2 or later of the General
|
|
* Public License.
|
|
*/
|
|
|
|
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
#include <stdint.h>
|
|
#include <stdbool.h>
|
|
#include <jansson.h>
|
|
#include <maxscale/buffer.h>
|
|
|
|
/** File magic and sync marker sizes block sizes */
|
|
#define AVRO_MAGIC_SIZE 4
|
|
#define SYNC_MARKER_SIZE 16
|
|
|
|
/** The file magic */
|
|
static const char avro_magic[] = {0x4f, 0x62, 0x6a, 0x01};
|
|
|
|
enum maxavro_value_type
|
|
{
|
|
MAXAVRO_TYPE_UNKNOWN = 0,
|
|
MAXAVRO_TYPE_INT,
|
|
MAXAVRO_TYPE_LONG,
|
|
MAXAVRO_TYPE_FLOAT,
|
|
MAXAVRO_TYPE_DOUBLE,
|
|
MAXAVRO_TYPE_BOOL,
|
|
MAXAVRO_TYPE_STRING,
|
|
MAXAVRO_TYPE_BYTES,
|
|
MAXAVRO_TYPE_ENUM,
|
|
MAXAVRO_TYPE_NULL,
|
|
MAXAVRO_TYPE_MAX
|
|
};
|
|
|
|
typedef struct
|
|
{
|
|
char *name;
|
|
void *extra;
|
|
enum maxavro_value_type type;
|
|
} MAXAVRO_SCHEMA_FIELD;
|
|
|
|
typedef struct
|
|
{
|
|
MAXAVRO_SCHEMA_FIELD *fields;
|
|
size_t num_fields;
|
|
} MAXAVRO_SCHEMA;
|
|
|
|
enum maxavro_codec
|
|
{
|
|
MAXAVRO_CODEC_NULL,
|
|
MAXAVRO_CODEC_DEFLATE,
|
|
MAXAVRO_CODEC_SNAPPY, /**< Not yet implemented */
|
|
};
|
|
|
|
enum maxavro_error
|
|
{
|
|
MAXAVRO_ERR_NONE,
|
|
MAXAVRO_ERR_IO,
|
|
MAXAVRO_ERR_MEMORY,
|
|
MAXAVRO_ERR_VALUE_OVERFLOW
|
|
};
|
|
|
|
typedef struct
|
|
{
|
|
FILE* file;
|
|
char* filename; /*< The filename */
|
|
MAXAVRO_SCHEMA* schema;
|
|
enum maxavro_codec codec;
|
|
uint64_t blocks_read; /*< Total number of data blocks read */
|
|
uint64_t records_read; /*< Total number of records read */
|
|
uint64_t bytes_read; /*< Total number of bytes read */
|
|
uint64_t records_in_block;
|
|
uint64_t records_read_from_block;
|
|
uint64_t bytes_read_from_block;
|
|
uint64_t buffer_size; /*< Size of the block in bytes */
|
|
uint8_t *buffer; /**< The uncompressed data */
|
|
uint8_t *buffer_end; /**< The byte after the end of the buffer*/
|
|
uint8_t *buffer_ptr; /**< Pointer to @c buffer which is moved as records are read */
|
|
/** The position @c ftell returns before the first record is read */
|
|
long header_end_pos;
|
|
long data_start_pos;
|
|
long block_start_pos;
|
|
bool metadata_read; /*< If datablock metadata has been read. This is kept
|
|
* in memory if EOF is reached but an attempt to read
|
|
* is made later when new data is available. We need
|
|
* to know when to read it and when not to. */
|
|
enum maxavro_error last_error; /*< Last error */
|
|
uint8_t sync[SYNC_MARKER_SIZE];
|
|
} MAXAVRO_FILE;
|
|
|
|
/** A record field value */
|
|
typedef union
|
|
{
|
|
uint64_t integer;
|
|
double floating;
|
|
char *string;
|
|
bool boolean;
|
|
void *bytes;
|
|
} MAXAVRO_RECORD_VALUE;
|
|
|
|
/** A record value */
|
|
typedef struct
|
|
{
|
|
MAXAVRO_SCHEMA_FIELD *field;
|
|
MAXAVRO_RECORD_VALUE *value;
|
|
size_t size;
|
|
} MAXAVRO_RECORD;
|
|
|
|
typedef struct
|
|
{
|
|
uint8_t *buffer; /*< Buffer memory */
|
|
size_t buffersize; /*< Size of the buffer */
|
|
size_t datasize; /*< size of written data */
|
|
uint64_t records; /*< Number of successfully written records */
|
|
MAXAVRO_FILE *avrofile; /*< The current open file */
|
|
} MAXAVRO_DATABLOCK;
|
|
|
|
typedef struct avro_map_value
|
|
{
|
|
char* key;
|
|
char* value;
|
|
struct avro_map_value *next;
|
|
struct avro_map_value *tail;
|
|
int blocks; /*< Number of added key-value blocks */
|
|
} MAXAVRO_MAP;
|
|
|
|
/** Opening and closing files */
|
|
MAXAVRO_FILE* maxavro_file_open(const char* filename);
|
|
void maxavro_file_close(MAXAVRO_FILE *file);
|
|
|
|
/** Reading records */
|
|
json_t* maxavro_record_read_json(MAXAVRO_FILE *file);
|
|
GWBUF* maxavro_record_read_binary(MAXAVRO_FILE *file);
|
|
|
|
/** Navigation of the file */
|
|
bool maxavro_record_seek(MAXAVRO_FILE *file, uint64_t offset);
|
|
bool maxavro_record_set_pos(MAXAVRO_FILE *file, long pos);
|
|
bool maxavro_next_block(MAXAVRO_FILE *file);
|
|
|
|
/** Get binary format header */
|
|
GWBUF* maxavro_file_binary_header(MAXAVRO_FILE *file);
|
|
|
|
/** File error functions */
|
|
enum maxavro_error maxavro_get_error(MAXAVRO_FILE *file);
|
|
const char* maxavro_get_error_string(MAXAVRO_FILE *file);
|
|
|
|
#endif
|