1/*
2 * This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 *
6 * Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V.
7 */
8
9#ifndef _STREAM_H_
10#define _STREAM_H_
11
12/*
13 * File: stream.h
14 * Auteur: Niels J. Nes
15 * Date: 09-01-2001
16 *
17 * Version 0.1: start
18 *
19 * This is the general interface to input/output. Each stream will
20 * contains some stream info (for now only byteorder). This is
21 * required for proper conversion on different byte order platforms.
22 */
23
24#include <unistd.h>
25#include <ctype.h>
26#include <stdio.h>
27
28#include <stdlib.h>
29#include <signal.h>
30#include <limits.h>
31
32/* avoid using "#ifdef WIN32" so that this file does not need our config.h */
33#if defined(_MSC_VER) || defined(__CYGWIN__) || defined(__MINGW32__)
34# ifndef LIBSTREAM
35# define stream_export extern __declspec(dllimport)
36# else
37# define stream_export extern __declspec(dllexport)
38# endif
39#else
40# define stream_export extern
41#endif
42#ifndef HAVE_HGE
43# ifdef HAVE___INT128
44# define HAVE_HGE 1
45typedef __int128 hge;
46# else
47# ifdef HAVE___INT128_T
48# define HAVE_HGE 1
49typedef __int128_t hge;
50# endif
51# endif
52#endif
53
54/* Defines to help the compiler check printf-style format arguments.
55 * These defines are also in our config.h, but we repeat them here so
56 * that we don't need that for this file.*/
57#if !defined(__GNUC__) || __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 5)
58/* This feature is available in gcc versions 2.5 and later. */
59# ifndef __attribute__
60# define __attribute__(Spec) /* empty */
61# endif
62#else
63/* The __-protected variants of `format' and `printf' attributes are
64 * accepted by gcc versions 2.6.4 (effectively 2.7) and later. */
65# if !defined(__format__) && (__GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 7))
66# define __format__ format
67# define __printf__ printf
68# endif
69#endif
70#if !defined(_MSC_VER) && !defined(_In_z_)
71# define _In_z_
72# define _Printf_format_string_
73#endif
74
75#define EOT 4
76
77/* fwf gets turned into a csv with these parameters */
78#define STREAM_FWF_FIELD_SEP '|'
79#define STREAM_FWF_ESCAPE '\\'
80#define STREAM_FWF_RECORD_SEP '\n'
81#define STREAM_FWF_FILLER ' '
82
83typedef struct stream stream;
84
85/* some os specific initialization */
86stream_export int mnstr_init(void);
87
88/* all mnstr_readX/mnstr_writeX return
89 * 0 on error
90 * !0 on success
91 */
92stream_export int mnstr_readBte(stream *restrict s, int8_t *restrict val);
93stream_export int mnstr_readChr(stream *restrict s, char *restrict val);
94stream_export int mnstr_writeChr(stream *s, char val);
95
96stream_export int mnstr_writeBte(stream *s, int8_t val);
97stream_export int mnstr_readSht(stream *restrict s, int16_t *restrict val);
98stream_export int mnstr_writeSht(stream *s, int16_t val);
99stream_export int mnstr_readInt(stream *restrict s, int *restrict val);
100stream_export int mnstr_writeInt(stream *s, int val);
101stream_export int mnstr_readLng(stream *restrict s, int64_t *restrict val);
102stream_export int mnstr_writeLng(stream *s, int64_t val);
103
104
105stream_export int mnstr_writeFlt(stream *s, float val);
106stream_export int mnstr_writeDbl(stream *s, double val);
107
108#ifdef HAVE_HGE
109stream_export int mnstr_readHge(stream *restrict s, hge *restrict val);
110stream_export int mnstr_writeHge(stream *s, hge val);
111#endif
112
113stream_export int mnstr_readBteArray(stream *restrict s, int8_t *restrict val, size_t cnt);
114stream_export int mnstr_writeBteArray(stream *restrict s, const int8_t *restrict val, size_t cnt);
115stream_export int mnstr_writeStr(stream *restrict s, const char *restrict val);
116stream_export int mnstr_readStr(stream *restrict s, char *restrict val);
117
118stream_export int mnstr_readShtArray(stream *restrict s, int16_t *restrict val, size_t cnt);
119stream_export int mnstr_writeShtArray(stream *restrict s, const int16_t *restrict val, size_t cnt);
120stream_export int mnstr_readIntArray(stream *restrict s, int *restrict val, size_t cnt);
121stream_export int mnstr_writeIntArray(stream *restrict s, const int *restrict val, size_t cnt);
122stream_export int mnstr_readLngArray(stream *restrict s, int64_t *restrict val, size_t cnt);
123stream_export int mnstr_writeLngArray(stream *restrict s, const int64_t *restrict val, size_t cnt);
124#ifdef HAVE_HGE
125stream_export int mnstr_readHgeArray(stream *restrict s, hge *restrict val, size_t cnt);
126stream_export int mnstr_writeHgeArray(stream *restrict s, const hge *restrict val, size_t cnt);
127#endif
128stream_export int mnstr_printf(stream *restrict s, _In_z_ _Printf_format_string_ const char *restrict format, ...)
129 __attribute__((__format__(__printf__, 2, 3)));
130stream_export ssize_t mnstr_read(stream *restrict s, void *restrict buf, size_t elmsize, size_t cnt);
131stream_export ssize_t mnstr_readline(stream *restrict s, void *restrict buf, size_t maxcnt);
132stream_export ssize_t mnstr_write(stream *restrict s, const void *restrict buf, size_t elmsize, size_t cnt);
133stream_export void mnstr_close(stream *s);
134stream_export void mnstr_destroy(stream *s);
135stream_export char *mnstr_error(stream *s);
136stream_export int mnstr_flush(stream *s);
137stream_export int mnstr_fsync(stream *s);
138stream_export int mnstr_fgetpos(stream *restrict s, fpos_t *restrict p);
139stream_export int mnstr_fsetpos(stream *restrict s, fpos_t *restrict p);
140stream_export char *mnstr_name(stream *s);
141stream_export int mnstr_errnr(stream *s);
142stream_export void mnstr_clearerr(stream *s);
143stream_export bool mnstr_isbinary(stream *s);
144stream_export bool mnstr_get_swapbytes(stream *s);
145stream_export void mnstr_set_bigendian(stream *s, bool bigendian);
146stream_export void mnstr_settimeout(stream *s, unsigned int ms, bool (*func)(void));
147stream_export int mnstr_isalive(stream *s);
148
149stream_export stream *open_rstream(const char *filename);
150stream_export stream *open_wstream(const char *filename);
151
152/* open in ascii stream in read mode */
153stream_export stream *open_rastream(const char *filename);
154
155/* open in ascii stream in write mode*/
156stream_export stream *open_wastream(const char *filename);
157
158stream_export void close_stream(stream *s);
159
160stream_export stream *open_urlstream(const char *url);
161
162stream_export stream *file_rstream(FILE *restrict fp, const char *restrict name);
163stream_export stream *file_wstream(FILE *restrict fp, const char *restrict name);
164stream_export stream *file_rastream(FILE *restrict fp, const char *restrict name);
165stream_export stream *file_wastream(FILE *restrict fp, const char *restrict name);
166
167stream_export FILE *getFile(stream *s);
168stream_export int getFileNo(stream *s); /* fileno(getFile(s)) */
169stream_export size_t getFileSize(stream *s);
170
171stream_export stream *iconv_rstream(stream *restrict ss, const char *restrict charset, const char *restrict name);
172stream_export stream *iconv_wstream(stream *restrict ss, const char *restrict charset, const char *restrict name);
173
174typedef struct buffer {
175 char *buf;
176 size_t pos;
177 size_t len;
178} buffer;
179
180stream_export void buffer_init(buffer *restrict b, char *restrict buf, size_t size);
181stream_export buffer *buffer_create(size_t size);
182stream_export char *buffer_get_buf(buffer *b);
183stream_export void buffer_destroy(buffer *b);
184
185stream_export stream *buffer_rastream(buffer *restrict b, const char *restrict name);
186stream_export stream *buffer_wastream(buffer *restrict b, const char *restrict name);
187stream_export buffer *mnstr_get_buffer(stream *s);
188
189/* note, the size is fixed to 8K, you cannot simply change it to any
190 * value */
191#define BLOCK (8 * 1024 - 2)
192
193/* Block stream is a stream which sends data in blocks of a known size
194 * (BLOCK size or dynamically changed using CHANGE_BLOCK_SIZE msg).
195 *
196 * A block is written once more than BLOCK size data has been written
197 * using the write commands or when the flush command is sent.
198 *
199 * All full blocks together with a single not full block form a major
200 * block. Major blocks can be used to synchronize the communication.
201 * Example server sends some reply, ie a major block consisting of
202 * various minor blocks. The header of the major block can contain
203 * special info which the client can interpret.
204 *
205 * Each read attempt tries to return the number of bytes. Once a lower
206 * number of bytes can be read the end of the major block is
207 * found. The next read will then start with a new major block.
208 */
209stream_export stream *block_stream(stream *s);
210stream_export bool isa_block_stream(stream *s);
211stream_export stream *bs_stream(stream *s);
212
213
214typedef enum {
215 PROTOCOL_AUTO = 0,
216 PROTOCOL_9 = 1,
217 PROTOCOL_10 = 2
218} protocol_version;
219
220typedef enum {
221 COMPRESSION_NONE = 0,
222 COMPRESSION_SNAPPY = 1,
223 COMPRESSION_LZ4 = 2,
224 COMPRESSION_AUTO = 255
225} compression_method;
226
227stream_export stream *block_stream2(stream *s, size_t bufsiz, compression_method comp);
228stream_export int bs2_resizebuf(stream *ss, size_t bufsiz);
229stream_export buffer bs2_buffer(stream *s);
230stream_export void bs2_setpos(stream *ss, size_t pos);
231
232
233/* read block of data including the end of block marker */
234stream_export ssize_t mnstr_read_block(stream *restrict s, void *restrict buf, size_t elmsize, size_t cnt);
235
236typedef struct bstream {
237 stream *s;
238 char *buf;
239 size_t size; /* size of buf */
240 size_t pos; /* the data cursor (ie read until pos) */
241 size_t len; /* len of the data (<= size) */
242 size_t mode; /* 0 line mode else size for block mode */
243 bool eof;
244} bstream;
245
246stream_export bstream *bstream_create(stream *rs, size_t chunk_size);
247stream_export void bstream_destroy(bstream *s);
248stream_export ssize_t bstream_read(bstream *s, size_t size);
249stream_export ssize_t bstream_next(bstream *s);
250
251typedef enum mnstr_errors {
252 MNSTR_NO__ERROR = 0,
253 MNSTR_OPEN_ERROR,
254 MNSTR_READ_ERROR,
255 MNSTR_WRITE_ERROR,
256 MNSTR_TIMEOUT
257} mnstr_errors;
258
259/* Callback stream is a read-only stream where the read function is
260 * provided by the caller. close and destroy are also provided. The
261 * private pointer is passed on to the callback functions when they
262 * are invoked. */
263stream_export stream *callback_stream(
264 void *restrict priv,
265 ssize_t (*read)(void *restrict priv, void *restrict buf, size_t elmsize, size_t cnt),
266 void (*close)(void *priv),
267 void (*destroy)(void *priv),
268 const char *restrict name);
269
270stream_export stream *stream_blackhole_create(void);
271
272stream_export stream *stream_fwf_create(stream *restrict s, size_t num_fields, size_t *restrict widths, char filler);
273
274#endif /*_STREAM_H_*/
275