1/*
2 * This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 *
6 * Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V.
7 */
8
9/*
10 * author N.J. Nes
11 */
12
13#include "monetdb_config.h"
14#include "sql_result.h"
15#include "str.h"
16#include "tablet.h"
17#include "mtime.h"
18#include "bat/res_table.h"
19#include "bat/bat_storage.h"
20#include "rel_exp.h"
21
22#ifndef HAVE_LLABS
23#define llabs(x) ((x) < 0 ? -(x) : (x))
24#endif
25
26#ifdef _MSC_VER
27/* use intrinsic functions on Windows */
28#define short_int_SWAP(s) ((short) _byteswap_ushort((unsigned short) (s)))
29/* on Windows, long is the same size as int */
30#define normal_int_SWAP(s) ((int) _byteswap_ulong((unsigned long) (s)))
31#define long_long_SWAP(s) ((lng) _byteswap_uint64((unsigned __int64) (s)))
32#else
33#define short_int_SWAP(s) ((short)(((0x00ff&(s))<<8) | ((0xff00&(s))>>8)))
34
35#define normal_int_SWAP(i) (((0x000000ff&(i))<<24) | ((0x0000ff00&(i))<<8) | \
36 ((0x00ff0000&(i))>>8) | ((0xff000000&(i))>>24))
37#define long_long_SWAP(l) \
38 ((((lng)normal_int_SWAP(l))<<32) | \
39 (0xffffffff&normal_int_SWAP(l>>32)))
40#endif
41
42#ifdef HAVE_HGE
43#define huge_int_SWAP(h) \
44 ((((hge)long_long_SWAP(h))<<64) | \
45 (0xffffffffffffffff&long_long_SWAP(h>>64)))
46#endif
47
48static lng
49mnstr_swap_lng(stream *s, lng lngval) {
50 return mnstr_get_swapbytes(s) ? long_long_SWAP(lngval) : lngval;
51}
52
53#define DEC_TOSTR(TYPE) \
54 do { \
55 char buf[64]; \
56 TYPE v = *(const TYPE *) a; \
57 int scale = (int) (ptrdiff_t) extra; \
58 int cur = 63, i, done = 0; \
59 int neg = v < 0; \
60 ssize_t l; \
61 if (is_##TYPE##_nil(v)) { \
62 if (*len < 5){ \
63 if (*Buf) \
64 GDKfree(*Buf); \
65 *len = 5; \
66 *Buf = GDKzalloc(*len); \
67 if (*Buf == NULL) { \
68 return -1; \
69 } \
70 } \
71 strcpy(*Buf, "NULL"); \
72 return 4; \
73 } \
74 if (v<0) \
75 v = -v; \
76 buf[cur--] = 0; \
77 if (scale){ \
78 for (i=0; i<scale; i++) { \
79 buf[cur--] = (char) (v%10 + '0'); \
80 v /= 10; \
81 } \
82 buf[cur--] = '.'; \
83 } \
84 while (v) { \
85 buf[cur--] = (char ) (v%10 + '0'); \
86 v /= 10; \
87 done = 1; \
88 } \
89 if (!done) \
90 buf[cur--] = '0'; \
91 if (neg) \
92 buf[cur--] = '-'; \
93 l = (64-cur-1); \
94 if ((ssize_t) *len < l){ \
95 if (*Buf) \
96 GDKfree(*Buf); \
97 *len = (size_t) l+1; \
98 *Buf = GDKzalloc(*len); \
99 if (*Buf == NULL) { \
100 return -1; \
101 } \
102 } \
103 strcpy(*Buf, buf+cur+1); \
104 return l-1; \
105 } while (0)
106
107static ssize_t
108dec_tostr(void *extra, char **Buf, size_t *len, int type, const void *a)
109{
110 /* support dec map to bte, sht, int and lng */
111 if (type == TYPE_bte) {
112 DEC_TOSTR(bte);
113 } else if (type == TYPE_sht) {
114 DEC_TOSTR(sht);
115 } else if (type == TYPE_int) {
116 DEC_TOSTR(int);
117 } else if (type == TYPE_lng) {
118 DEC_TOSTR(lng);
119#ifdef HAVE_HGE
120 } else if (type == TYPE_hge) {
121 DEC_TOSTR(hge);
122#endif
123 } else {
124 GDKerror("Decimal cannot be mapped to %s\n", ATOMname(type));
125 }
126 return -1;
127}
128
129struct time_res {
130 int fraction;
131 int has_tz;
132 lng timezone;
133};
134
135static ssize_t
136sql_time_tostr(void *TS_RES, char **buf, size_t *len, int type, const void *A)
137{
138 struct time_res *ts_res = TS_RES;
139 ssize_t len1;
140 size_t big = 128;
141 char buf1[128], *s1 = buf1, *s;
142 daytime tmp;
143
144 (void) type;
145 tmp = *(const daytime *) A;
146 if (ts_res->has_tz)
147 tmp = daytime_add_usec_modulo(tmp, ts_res->timezone * 1000);
148
149 len1 = daytime_precision_tostr(&s1, &big, tmp, ts_res->fraction, true);
150 if (len1 < 0)
151 return -1;
152 if (len1 == 3 && strcmp(s1, "nil") == 0) {
153 if (*len < 4 || *buf == NULL) {
154 GDKfree(*buf);
155 *buf = GDKzalloc(*len = 4);
156 if (*buf == NULL)
157 return -1;
158 }
159 strcpy(*buf, "nil");
160 return len1;
161 }
162
163 if (*len < (size_t) len1 + 8) {
164 if (*buf)
165 GDKfree(*buf);
166 *buf = (str) GDKzalloc(*len = len1 + 8);
167 if (*buf == NULL) {
168 return -1;
169 }
170 }
171 s = *buf;
172 strcpy(s, buf1);
173 s += len1;
174
175 if (ts_res->has_tz) {
176 lng timezone = llabs(ts_res->timezone / 60000);
177 s += sprintf(s, "%c%02d:%02d",
178 (ts_res->timezone >= 0) ? '+' : '-',
179 (int) (timezone / 60), (int) (timezone % 60));
180 }
181 return (ssize_t) (s - *buf);
182}
183
184static ssize_t
185sql_timestamp_tostr(void *TS_RES, char **buf, size_t *len, int type, const void *A)
186{
187 struct time_res *ts_res = TS_RES;
188 ssize_t len1, len2;
189 size_t big = 128;
190 char buf1[128], buf2[128], *s, *s1 = buf1, *s2 = buf2;
191 timestamp tmp;
192 lng timezone = ts_res->timezone;
193 date days;
194 daytime usecs;
195
196 (void) type;
197 tmp = *(const timestamp *)A;
198 if (ts_res->has_tz) {
199 tmp = timestamp_add_usec(tmp, timezone * 1000);
200 }
201 days = timestamp_date(tmp);
202 usecs = timestamp_daytime(tmp);
203 len1 = date_tostr(&s1, &big, &days, true);
204 len2 = daytime_precision_tostr(&s2, &big, usecs, ts_res->fraction, true);
205 if (len1 < 0 || len2 < 0) {
206 GDKfree(s1);
207 GDKfree(s2);
208 return -1;
209 }
210
211 if ((len1 == 3 && strcmp(s1, "nil") == 0) ||
212 (len2 == 3 && strcmp(s2, "nil") == 0)) {
213 if (*len < 4 || *buf == NULL) {
214 GDKfree(*buf);
215 *buf = GDKzalloc(*len = 4);
216 if (*buf == NULL)
217 return -1;
218 }
219 strcpy(*buf, "nil");
220 return len1;
221 }
222
223 if (*len < (size_t) len1 + (size_t) len2 + 8) {
224 if (*buf)
225 GDKfree(*buf);
226 *buf = (str) GDKzalloc(*len = (size_t) (len1 + len2 + 8));
227 if (*buf == NULL) {
228 return -1;
229 }
230 }
231 s = *buf;
232 strcpy(s, buf1);
233 s += len1;
234 *s++ = ' ';
235 strcpy(s, buf2);
236 s += len2;
237 s[0] = 0;
238
239 if (ts_res->has_tz) {
240 timezone = ts_res->timezone / 60000;
241 *s++ = (ts_res->timezone >= 0) ? '+' : '-';
242 sprintf(s, "%02d:%02d", (int) (llabs(timezone) / 60), (int) (llabs(timezone) % 60));
243 s += 5;
244 }
245 return (ssize_t) (s - *buf);
246}
247
248static int
249STRwidth(const char *s)
250{
251 int len = 0;
252 int c;
253 int n;
254
255 if (GDK_STRNIL(s))
256 return int_nil;
257 c = 0;
258 n = 0;
259 while (*s != 0) {
260 if ((*s & 0x80) == 0) {
261 assert(n == 0);
262 len++;
263 n = 0;
264 } else if ((*s & 0xC0) == 0x80) {
265 c = (c << 6) | (*s & 0x3F);
266 if (--n == 0) {
267 /* last byte of a multi-byte character */
268 len++;
269 /* this list was created by combining
270 * the code points marked as
271 * Emoji_Presentation in
272 * /usr/share/unicode/emoji/emoji-data.txt
273 * and code points marked either F or
274 * W in EastAsianWidth.txt; this list
275 * is up-to-date with Unicode 9.0 */
276 if ((0x1100 <= c && c <= 0x115F) ||
277 (0x231A <= c && c <= 0x231B) ||
278 (0x2329 <= c && c <= 0x232A) ||
279 (0x23E9 <= c && c <= 0x23EC) ||
280 c == 0x23F0 ||
281 c == 0x23F3 ||
282 (0x25FD <= c && c <= 0x25FE) ||
283 (0x2614 <= c && c <= 0x2615) ||
284 (0x2648 <= c && c <= 0x2653) ||
285 c == 0x267F ||
286 c == 0x2693 ||
287 c == 0x26A1 ||
288 (0x26AA <= c && c <= 0x26AB) ||
289 (0x26BD <= c && c <= 0x26BE) ||
290 (0x26C4 <= c && c <= 0x26C5) ||
291 c == 0x26CE ||
292 c == 0x26D4 ||
293 c == 0x26EA ||
294 (0x26F2 <= c && c <= 0x26F3) ||
295 c == 0x26F5 ||
296 c == 0x26FA ||
297 c == 0x26FD ||
298 c == 0x2705 ||
299 (0x270A <= c && c <= 0x270B) ||
300 c == 0x2728 ||
301 c == 0x274C ||
302 c == 0x274E ||
303 (0x2753 <= c && c <= 0x2755) ||
304 c == 0x2757 ||
305 (0x2795 <= c && c <= 0x2797) ||
306 c == 0x27B0 ||
307 c == 0x27BF ||
308 (0x2B1B <= c && c <= 0x2B1C) ||
309 c == 0x2B50 ||
310 c == 0x2B55 ||
311 (0x2E80 <= c && c <= 0x2E99) ||
312 (0x2E9B <= c && c <= 0x2EF3) ||
313 (0x2F00 <= c && c <= 0x2FD5) ||
314 (0x2FF0 <= c && c <= 0x2FFB) ||
315 (0x3000 <= c && c <= 0x303E) ||
316 (0x3041 <= c && c <= 0x3096) ||
317 (0x3099 <= c && c <= 0x30FF) ||
318 (0x3105 <= c && c <= 0x312D) ||
319 (0x3131 <= c && c <= 0x318E) ||
320 (0x3190 <= c && c <= 0x31BA) ||
321 (0x31C0 <= c && c <= 0x31E3) ||
322 (0x31F0 <= c && c <= 0x321E) ||
323 (0x3220 <= c && c <= 0x3247) ||
324 (0x3250 <= c && c <= 0x32FE) ||
325 (0x3300 <= c && c <= 0x4DBF) ||
326 (0x4E00 <= c && c <= 0xA48C) ||
327 (0xA490 <= c && c <= 0xA4C6) ||
328 (0xA960 <= c && c <= 0xA97C) ||
329 (0xAC00 <= c && c <= 0xD7A3) ||
330 (0xF900 <= c && c <= 0xFAFF) ||
331 (0xFE10 <= c && c <= 0xFE19) ||
332 (0xFE30 <= c && c <= 0xFE52) ||
333 (0xFE54 <= c && c <= 0xFE66) ||
334 (0xFE68 <= c && c <= 0xFE6B) ||
335 (0xFF01 <= c && c <= 0xFF60) ||
336 (0xFFE0 <= c && c <= 0xFFE6) ||
337 c == 0x16FE0 ||
338 (0x17000 <= c && c <= 0x187EC) ||
339 (0x18800 <= c && c <= 0x18AF2) ||
340 (0x1B000 <= c && c <= 0x1B001) ||
341 c == 0x1F004 ||
342 c == 0x1F0CF ||
343 c == 0x1F18E ||
344 (0x1F191 <= c && c <= 0x1F19A) ||
345 /* removed 0x1F1E6..0x1F1FF */
346 (0x1F200 <= c && c <= 0x1F202) ||
347 (0x1F210 <= c && c <= 0x1F23B) ||
348 (0x1F240 <= c && c <= 0x1F248) ||
349 (0x1F250 <= c && c <= 0x1F251) ||
350 (0x1F300 <= c && c <= 0x1F320) ||
351 (0x1F32D <= c && c <= 0x1F335) ||
352 (0x1F337 <= c && c <= 0x1F37C) ||
353 (0x1F37E <= c && c <= 0x1F393) ||
354 (0x1F3A0 <= c && c <= 0x1F3CA) ||
355 (0x1F3CF <= c && c <= 0x1F3D3) ||
356 (0x1F3E0 <= c && c <= 0x1F3F0) ||
357 c == 0x1F3F4 ||
358 (0x1F3F8 <= c && c <= 0x1F43E) ||
359 c == 0x1F440 ||
360 (0x1F442 <= c && c <= 0x1F4FC) ||
361 (0x1F4FF <= c && c <= 0x1F53D) ||
362 (0x1F54B <= c && c <= 0x1F54E) ||
363 (0x1F550 <= c && c <= 0x1F567) ||
364 c == 0x1F57A ||
365 (0x1F595 <= c && c <= 0x1F596) ||
366 c == 0x1F5A4 ||
367 (0x1F5FB <= c && c <= 0x1F64F) ||
368 (0x1F680 <= c && c <= 0x1F6C5) ||
369 c == 0x1F6CC ||
370 (0x1F6D0 <= c && c <= 0x1F6D2) ||
371 (0x1F6EB <= c && c <= 0x1F6EC) ||
372 (0x1F6F4 <= c && c <= 0x1F6F6) ||
373 (0x1F910 <= c && c <= 0x1F91E) ||
374 (0x1F920 <= c && c <= 0x1F927) ||
375 c == 0x1F930 ||
376 (0x1F933 <= c && c <= 0x1F93E) ||
377 (0x1F940 <= c && c <= 0x1F94B) ||
378 (0x1F950 <= c && c <= 0x1F95E) ||
379 (0x1F980 <= c && c <= 0x1F991) ||
380 c == 0x1F9C0 ||
381 (0x20000 <= c && c <= 0x2FFFD) ||
382 (0x30000 <= c && c <= 0x3FFFD))
383 len++;
384 }
385 } else if ((*s & 0xE0) == 0xC0) {
386 assert(n == 0);
387 n = 1;
388 c = *s & 0x1F;
389 } else if ((*s & 0xF0) == 0xE0) {
390 assert(n == 0);
391 n = 2;
392 c = *s & 0x0F;
393 } else if ((*s & 0xF8) == 0xF0) {
394 assert(n == 0);
395 n = 3;
396 c = *s & 0x07;
397 } else if ((*s & 0xFC) == 0xF8) {
398 assert(n == 0);
399 n = 4;
400 c = *s & 0x03;
401 } else {
402 assert(0);
403 n = 0;
404 }
405 s++;
406 }
407 return len;
408}
409
410static int
411bat_max_strlength(BAT *b)
412{
413 BUN p, q;
414 int l = 0;
415 int max = 0;
416 BATiter bi = bat_iterator(b);
417
418 BATloop(b, p, q) {
419 l = STRwidth((const char *) BUNtvar(bi, p));
420
421 if (is_int_nil(l))
422 l = 0;
423 if (l > max)
424 max = l;
425 }
426 return max;
427}
428
429static size_t
430bat_max_btelength(BAT *b)
431{
432 BUN p, q;
433 lng max = 0;
434 lng min = 0;
435 size_t ret = 0;
436 const bte *vals = (const bte *) Tloc(b, 0);
437
438 BATloop(b, p, q) {
439 lng m = 0;
440 bte l = vals[p];
441
442 if (!is_bte_nil(l))
443 m = l;
444 if (m > max)
445 max = m;
446 if (m < min)
447 min = m;
448 }
449
450 if (-min > max / 10) {
451 max = -min;
452 ret++; /* '-' */
453 }
454 while (max /= 10)
455 ret++;
456 ret++;
457 return ret;
458}
459
460static size_t
461bat_max_shtlength(BAT *b)
462{
463 BUN p, q;
464 lng max = 0;
465 lng min = 0;
466 size_t ret = 0;
467 const sht *vals = (const sht *) Tloc(b, 0);
468
469 BATloop(b, p, q) {
470 lng m = 0;
471 sht l = vals[p];
472
473 if (!is_sht_nil(l))
474 m = l;
475 if (m > max)
476 max = m;
477 if (m < min)
478 min = m;
479 }
480
481 if (-min > max / 10) {
482 max = -min;
483 ret++; /* '-' */
484 }
485 while (max /= 10)
486 ret++;
487 ret++;
488 return ret;
489}
490
491static size_t
492bat_max_intlength(BAT *b)
493{
494 BUN p, q;
495 lng max = 0;
496 lng min = 0;
497 size_t ret = 0;
498 const int *vals = (const int *) Tloc(b, 0);
499
500 BATloop(b, p, q) {
501 lng m = 0;
502 int l = vals[p];
503
504 if (!is_int_nil(l))
505 m = l;
506 if (m > max)
507 max = m;
508 if (m < min)
509 min = m;
510 }
511
512 if (-min > max / 10) {
513 max = -min;
514 ret++; /* '-' */
515 }
516 while (max /= 10)
517 ret++;
518 ret++;
519 return ret;
520}
521
522static size_t
523bat_max_lnglength(BAT *b)
524{
525 BUN p, q;
526 lng max = 0;
527 lng min = 0;
528 size_t ret = 0;
529 const lng *vals = (const lng *) Tloc(b, 0);
530
531 BATloop(b, p, q) {
532 lng m = 0;
533 lng l = vals[p];
534
535 if (!is_lng_nil(l))
536 m = l;
537 if (m > max)
538 max = m;
539 if (m < min)
540 min = m;
541 }
542
543 if (-min > max / 10) {
544 max = -min;
545 ret++; /* '-' */
546 }
547 while (max /= 10)
548 ret++;
549 ret++;
550 return ret;
551}
552
553#ifdef HAVE_HGE
554static size_t
555bat_max_hgelength(BAT *b)
556{
557 BUN p, q;
558 hge max = 0;
559 hge min = 0;
560 size_t ret = 0;
561 const hge *vals = (const hge *) Tloc(b, 0);
562
563 BATloop(b, p, q) {
564 hge m = 0;
565 hge l = vals[p];
566
567 if (!is_hge_nil(l))
568 m = l;
569 if (m > max) max = m;
570 if (m < min) min = m;
571 }
572
573 if (-min > max / 10) {
574 max = -min;
575 ret++; /* '-' */
576 }
577 while (max /= 10)
578 ret++;
579 ret++;
580 return ret;
581}
582#endif
583
584#define DEC_FRSTR(X) \
585 do { \
586 sql_column *col = c->extra; \
587 sql_subtype *t = &col->type; \
588 \
589 unsigned int i, neg = 0; \
590 X *r; \
591 X res = 0; \
592 while(isspace((unsigned char) *s)) \
593 s++; \
594 if (*s == '-'){ \
595 neg = 1; \
596 s++; \
597 } else if (*s == '+'){ \
598 neg = 0; \
599 s++; \
600 } \
601 for (i = 0; *s && *s != '.' && ((res == 0 && *s == '0') || i < t->digits - t->scale); s++) { \
602 if (!*s || !isdigit((unsigned char) *s)) \
603 return NULL; \
604 res *= 10; \
605 res += (*s-'0'); \
606 if (res) \
607 i++; \
608 } \
609 if (!*s && t->scale) { \
610 for( i = 0; i < t->scale; i++) { \
611 res *= 10; \
612 } \
613 } \
614 while(isspace((unsigned char) *s)) \
615 s++; \
616 if (*s) { \
617 if (*s != '.') \
618 return NULL; \
619 s++; \
620 for (i = 0; *s && isdigit((unsigned char) *s) && i < t->scale; i++, s++) { \
621 res *= 10; \
622 res += *s - '0'; \
623 } \
624 while(isspace((unsigned char) *s)) \
625 s++; \
626 for (; i < t->scale; i++) { \
627 res *= 10; \
628 } \
629 } \
630 if (*s) \
631 return NULL; \
632 r = c->data; \
633 if (r == NULL && \
634 (r = GDKzalloc(sizeof(X))) == NULL) \
635 return NULL; \
636 c->data = r; \
637 if (neg) \
638 *r = -res; \
639 else \
640 *r = res; \
641 return (void *) r; \
642 } while (0)
643
644static void *
645dec_frstr(Column *c, int type, const char *s)
646{
647 /* support dec map to bte, sht, int and lng */
648 if( strcmp(s,"nil")== 0)
649 return NULL;
650 if (type == TYPE_bte) {
651 DEC_FRSTR(bte);
652 } else if (type == TYPE_sht) {
653 DEC_FRSTR(sht);
654 } else if (type == TYPE_int) {
655 DEC_FRSTR(int);
656 } else if (type == TYPE_lng) {
657 DEC_FRSTR(lng);
658#ifdef HAVE_HGE
659 } else if (type == TYPE_hge) {
660 DEC_FRSTR(hge);
661#endif
662 }
663 return NULL;
664}
665
666static void *
667sec_frstr(Column *c, int type, const char *s)
668{
669 /* read a sec_interval value
670 * this knows that the stored scale is always 3 */
671 unsigned int i, neg = 0;
672 lng *r;
673 lng res = 0;
674
675 (void) c;
676 (void) type;
677 assert(type == TYPE_lng);
678
679 if (*s == '-') {
680 neg = 1;
681 s++;
682 } else if (*s == '+') {
683 neg = 0;
684 s++;
685 }
686 for (i = 0; i < (19 - 3) && *s && *s != '.'; i++, s++) {
687 if (!isdigit((unsigned char) *s))
688 return NULL;
689 res *= 10;
690 res += (*s - '0');
691 }
692 i = 0;
693 if (*s) {
694 if (*s != '.')
695 return NULL;
696 s++;
697 for (; *s && i < 3; i++, s++) {
698 if (!isdigit((unsigned char) *s))
699 return NULL;
700 res *= 10;
701 res += (*s - '0');
702 }
703 }
704 if (*s)
705 return NULL;
706 for (; i < 3; i++) {
707 res *= 10;
708 }
709 r = c->data;
710 if (r == NULL && (r = (lng *) GDKzalloc(sizeof(lng))) == NULL)
711 return NULL;
712 c->data = r;
713 if (neg)
714 *r = -res;
715 else
716 *r = res;
717 return (void *) r;
718}
719
720/* Literal parsing for SQL all pass through this routine */
721static void *
722_ASCIIadt_frStr(Column *c, int type, const char *s)
723{
724 ssize_t len;
725 const char *e;
726
727 if (type == TYPE_str) {
728 sql_column *col = (sql_column *) c->extra;
729 int slen;
730
731 for (e = s; *e; e++)
732 ;
733 len = (ssize_t) (e - s + 1);
734
735 /* or shouldn't len rather be ssize_t, here? */
736
737 if ((ssize_t) c->len < len) {
738 void *p;
739 c->len = (size_t) len;
740 if ((p = GDKrealloc(c->data, c->len)) == NULL) {
741 GDKfree(c->data);
742 c->data = NULL;
743 c->len = 0;
744 return NULL;
745 }
746 c->data = p;
747 }
748 if (s == e || *s == 0) {
749 len = -1;
750 *(char *) c->data = 0;
751 } else if ((len = GDKstrFromStr(c->data, (unsigned char *) s, (ssize_t) (e - s))) < 0) {
752 return NULL;
753 }
754 s = c->data;
755 STRLength(&slen, (const str *) &s);
756 if (col->type.digits > 0 && len > 0 && slen > (int) col->type.digits) {
757 len = STRwidth(c->data);
758 if (len > (ssize_t) col->type.digits)
759 return NULL;
760 }
761 return c->data;
762 }
763 // All other values are not allowed to the MonetDB nil value
764 if( strcmp(s,"nil")== 0)
765 return NULL;
766
767 len = (*BATatoms[type].atomFromStr) (s, &c->len, &c->data, true);
768 if (len < 0)
769 return NULL;
770 if (len == 0 || s[len]) {
771 /* decimals can be converted to integers when *.000 */
772 if (s[len++] == '.')
773 switch (type) {
774 case TYPE_bte:
775 case TYPE_int:
776 case TYPE_lng:
777 case TYPE_sht:
778#ifdef HAVE_HGE
779 case TYPE_hge:
780#endif
781 while (s[len] == '0')
782 len++;
783 if (s[len] == 0)
784 return c->data;
785 }
786 return NULL;
787 }
788 return c->data;
789}
790
791
792static ssize_t
793_ASCIIadt_toStr(void *extra, char **buf, size_t *len, int type, const void *a)
794{
795 if (type == TYPE_str) {
796 Column *c = extra;
797 char *dst;
798 const char *src = a;
799 size_t l = escapedStrlen(src, c->sep, c->rsep, c->quote), l2 = 0;
800
801 if (c->quote)
802 l = escapedStrlen(src, NULL, NULL, c->quote);
803 else
804 l = escapedStrlen(src, c->sep, c->rsep, 0);
805 if (l + 3 > *len) {
806 GDKfree(*buf);
807 *len = 2 * l + 3;
808 *buf = GDKzalloc(*len);
809 if (*buf == NULL) {
810 return -1;
811 }
812 }
813 dst = *buf;
814 if (c->quote) {
815 dst[0] = c->quote;
816 l2 = 1;
817 l = escapedStr(dst + l2, src, *len - l2, NULL, NULL, c->quote);
818 } else {
819 l = escapedStr(dst + l2, src, *len - l2, c->sep, c->rsep, 0);
820 }
821 if (l2) {
822 dst[l + l2] = c->quote;
823 l2++;
824 }
825 dst[l + l2] = 0;
826 return l + l2;
827 } else {
828 return (*BATatoms[type].atomToStr) (buf, len, a, true);
829 }
830}
831
832
833static int
834has_whitespace(const char *s)
835{
836 if (*s == ' ' || *s == '\t')
837 return 1;
838 while (*s)
839 s++;
840 s--;
841 if (*s == ' ' || *s == '\t')
842 return 1;
843 return 0;
844}
845
846str
847mvc_import_table(Client cntxt, BAT ***bats, mvc *m, bstream *bs, sql_table *t, const char *sep, const char *rsep, const char *ssep, const char *ns, lng sz, lng offset, int locked, int best, bool from_stdin)
848{
849 int i = 0, j;
850 node *n;
851 Tablet as;
852 Column *fmt;
853 BUN cnt = 0;
854 str msg = MAL_SUCCEED;
855
856 *bats =0; // initialize the receiver
857
858 if (!bs) {
859 sql_error(m, 500, "no stream (pointer) provided");
860 return NULL;
861 }
862 if (mnstr_errnr(bs->s)) {
863 sql_error(m, 500, "stream not open %d", mnstr_errnr(bs->s));
864 return NULL;
865 }
866 if (offset < 0 || offset > (lng) BUN_MAX) {
867 sql_error(m, 500, "offset out of range");
868 return NULL;
869 }
870
871 if (locked) {
872 /* flush old changes to disk */
873 sql_trans_end(m->session);
874 store_apply_deltas(true);
875 sql_trans_begin(m->session);
876 }
877
878 if (offset > 0)
879 offset--;
880 if (t->columns.set) {
881 stream *out = m->scanner.ws;
882
883 as = (Tablet) {
884 .nr_attrs = list_length(t->columns.set),
885 .nr = (sz < 1) ? BUN_NONE : (BUN) sz,
886 .offset = (BUN) offset,
887 .error = NULL,
888 .tryall = 0,
889 .complaints = NULL,
890 .filename = m->scanner.rs == bs ? NULL : "",
891 };
892 fmt = GDKzalloc(sizeof(Column) * (as.nr_attrs + 1));
893 if (fmt == NULL) {
894 sql_error(m, 500, SQLSTATE(HY001) MAL_MALLOC_FAIL);
895 return NULL;
896 }
897 as.format = fmt;
898 if (!isa_block_stream(bs->s))
899 out = NULL;
900
901 for (n = t->columns.set->h, i = 0; n; n = n->next, i++) {
902 sql_column *col = n->data;
903
904 fmt[i].name = col->base.name;
905 fmt[i].sep = (n->next) ? sep : rsep;
906 fmt[i].rsep = rsep;
907 fmt[i].seplen = _strlen(fmt[i].sep);
908 fmt[i].type = sql_subtype_string(&col->type);
909 fmt[i].adt = ATOMindex(col->type.type->base.name);
910 fmt[i].tostr = &_ASCIIadt_toStr;
911 fmt[i].frstr = &_ASCIIadt_frStr;
912 fmt[i].extra = col;
913 fmt[i].len = ATOMlen(fmt[i].adt, ATOMnilptr(fmt[i].adt));
914 fmt[i].data = GDKzalloc(fmt[i].len);
915 if(fmt[i].data == NULL || fmt[i].type == NULL) {
916 for (j = 0; j < i; j++) {
917 GDKfree(fmt[j].type);
918 GDKfree(fmt[j].data);
919 BBPunfix(fmt[j].c->batCacheid);
920 }
921 GDKfree(fmt[i].type);
922 GDKfree(fmt[i].data);
923 sql_error(m, 500, SQLSTATE(HY001) "failed to allocate space for column");
924 return NULL;
925 }
926 fmt[i].c = NULL;
927 fmt[i].ws = !has_whitespace(fmt[i].sep);
928 fmt[i].quote = ssep ? ssep[0] : 0;
929 fmt[i].nullstr = ns;
930 fmt[i].null_length = strlen(ns);
931 fmt[i].nildata = ATOMnilptr(fmt[i].adt);
932 fmt[i].skip = (col->base.name[0] == '%');
933 if (col->type.type->eclass == EC_DEC) {
934 fmt[i].tostr = &dec_tostr;
935 fmt[i].frstr = &dec_frstr;
936 } else if (col->type.type->eclass == EC_SEC) {
937 fmt[i].tostr = &dec_tostr;
938 fmt[i].frstr = &sec_frstr;
939 }
940 fmt[i].size = ATOMsize(fmt[i].adt);
941
942 if (locked) {
943 BAT *b = store_funcs.bind_col(m->session->tr, col, RDONLY);
944 if (b == NULL) {
945 for (j = 0; j < i; j++) {
946 GDKfree(fmt[j].type);
947 GDKfree(fmt[j].data);
948 BBPunfix(fmt[j].c->batCacheid);
949 }
950 GDKfree(fmt[i].type);
951 GDKfree(fmt[i].data);
952 sql_error(m, 500, "failed to bind to table column");
953 return NULL;
954 }
955
956 HASHdestroy(b);
957
958 fmt[i].c = b;
959 cnt = BATcount(b);
960 if (sz > 0 && BATcapacity(b) < (BUN) sz) {
961 if (BATextend(fmt[i].c, (BUN) sz) != GDK_SUCCEED) {
962 for (j = 0; j <= i; j++) {
963 GDKfree(fmt[j].type);
964 GDKfree(fmt[j].data);
965 BBPunfix(fmt[j].c->batCacheid);
966 }
967 sql_error(m, 500, SQLSTATE(HY001) "failed to allocate space for column");
968 return NULL;
969 }
970 }
971 fmt[i].ci = bat_iterator(fmt[i].c);
972 fmt[i].c->batDirtydesc = true;
973 }
974 }
975 if ( (locked || (msg = TABLETcreate_bats(&as, (BUN) (sz < 0 ? 1000 : sz))) == MAL_SUCCEED) ){
976 if (!sz || (SQLload_file(cntxt, &as, bs, out, sep, rsep, ssep ? ssep[0] : 0, offset, sz, best, from_stdin, t->base.name) != BUN_NONE &&
977 (best || !as.error))) {
978 *bats = (BAT**) GDKzalloc(sizeof(BAT *) * as.nr_attrs);
979 if ( *bats == NULL){
980 sql_error(m, 500, SQLSTATE(HY001) "failed to allocate space for column");
981 TABLETdestroy_format(&as);
982 return NULL;
983 }
984 if (locked)
985 msg = TABLETcollect_parts(*bats,&as, cnt);
986 else
987 msg = TABLETcollect(*bats,&as);
988 } else if (locked) { /* restore old counts */
989 for (n = t->columns.set->h, i = 0; n; n = n->next, i++) {
990 sql_column *col = n->data;
991 BAT *b = store_funcs.bind_col(m->session->tr, col, RDONLY);
992 if (b == NULL)
993 sql_error(m, 500, "failed to bind to temporary column");
994 else {
995 BATsetcount(b, cnt);
996 BBPunfix(b->batCacheid);
997 }
998 }
999 }
1000 }
1001 if (locked) { /* fix delta structures and transaction */
1002 for (n = t->columns.set->h, i = 0; n; n = n->next, i++) {
1003 sql_column *c = n->data;
1004 BAT *b = store_funcs.bind_col(m->session->tr, c, RDONLY);
1005 sql_delta *d = c->data;
1006
1007 c->base.wtime = t->base.wtime = t->s->base.wtime = m->session->tr->wtime = m->session->tr->wstime;
1008 if ( b == NULL)
1009 sql_error(m, 500, "failed to bind to delta column");
1010 else {
1011 d->ibase = (oid) (d->cnt = BATcount(b));
1012 BBPunfix(b->batCacheid);
1013 }
1014 }
1015 }
1016 if (as.error) {
1017 if( !best) sql_error(m, 500, "%s", getExceptionMessage(as.error));
1018 freeException(as.error);
1019 as.error = NULL;
1020 }
1021 for (n = t->columns.set->h, i = 0; n; n = n->next, i++) {
1022 fmt[i].sep = NULL;
1023 fmt[i].rsep = NULL;
1024 fmt[i].nullstr = NULL;
1025 }
1026 TABLETdestroy_format(&as);
1027 }
1028 return msg;
1029}
1030
1031/*
1032 * mvc_export_result dumps the sql header information and the
1033 * first part (reply_size) of the result set. It should be produced in Monet format to
1034 * enable mapi to work with it.
1035 */
1036
1037static int
1038mvc_export_warning(stream *s, str w)
1039{
1040 str tmp = NULL;
1041 while (w != NULL && *w != '\0') {
1042 if ((tmp = strchr(w, (int) '\n')) != NULL)
1043 *tmp++ = '\0';
1044 if (mnstr_printf(s, "#%s", w) < 0)
1045 return (-1);
1046 w = tmp;
1047 }
1048 return (1);
1049}
1050
1051int
1052mvc_export_prepare(mvc *c, stream *out, cq *q, str w)
1053{
1054 node *n;
1055 int nparam = c->params ? list_length(c->params) : 0;
1056 int nrows = nparam;
1057 size_t len1 = 0, len4 = 0, len5 = 0, len6 = 0; /* column widths */
1058 int len2 = 1, len3 = 1;
1059 sql_arg *a;
1060 sql_subtype *t;
1061 sql_rel *r = q->rel;
1062
1063 if (!out)
1064 return 0;
1065
1066 if (is_topn(r->op))
1067 r = r->l;
1068 if (r && is_project(r->op) && r->exps) {
1069 unsigned int max2 = 10, max3 = 10; /* to help calculate widths */
1070 nrows += list_length(r->exps);
1071
1072 for (n = r->exps->h; n; n = n->next) {
1073 const char *name;
1074 sql_exp *e = n->data;
1075 size_t slen;
1076
1077 t = exp_subtype(e);
1078 slen = strlen(t->type->sqlname);
1079 if (slen > len1)
1080 len1 = slen;
1081 while (t->digits >= max2) {
1082 len2++;
1083 max2 *= 10;
1084 }
1085 while (t->scale >= max3) {
1086 len3++;
1087 max3 *= 10;
1088 }
1089 name = exp_relname(e);
1090 if (!name && e->type == e_column && e->l)
1091 name = e->l;
1092 slen = name ? strlen(name) : 0;
1093 if (slen > len5)
1094 len5 = slen;
1095 name = exp_name(e);
1096 if (!name && e->type == e_column && e->r)
1097 name = e->r;
1098 slen = name ? strlen(name) : 0;
1099 if (slen > len6)
1100 len6 = slen;
1101 }
1102 }
1103 /* calculate column widths */
1104 if (c->params) {
1105 unsigned int max2 = 10, max3 = 10; /* to help calculate widths */
1106
1107 for (n = c->params->h; n; n = n->next) {
1108 size_t slen;
1109
1110 a = n->data;
1111 t = &a->type;
1112 slen = strlen(t->type->sqlname);
1113 if (slen > len1)
1114 len1 = slen;
1115 while (t->digits >= max2) {
1116 len2++;
1117 max2 *= 10;
1118 }
1119 while (t->scale >= max3) {
1120 len3++;
1121 max3 *= 10;
1122 }
1123
1124 }
1125 }
1126
1127 /* write header, query type: Q_PREPARE */
1128 if (mnstr_printf(out, "&5 %d %d 6 %d\n" /* TODO: add type here: r(esult) or u(pdate) */
1129 "%% .prepare,\t.prepare,\t.prepare,\t.prepare,\t.prepare,\t.prepare # table_name\n" "%% type,\tdigits,\tscale,\tschema,\ttable,\tcolumn # name\n" "%% varchar,\tint,\tint,\tstr,\tstr,\tstr # type\n" "%% %zu,\t%d,\t%d,\t"
1130 "%zu,\t%zu,\t%zu # length\n", q->id, nrows, nrows, len1, len2, len3, len4, len5, len6) < 0) {
1131 return -1;
1132 }
1133
1134 if (r && is_project(r->op) && r->exps) {
1135 for (n = r->exps->h; n; n = n->next) {
1136 const char *name, *rname, *schema = NULL;
1137 sql_exp *e = n->data;
1138
1139 t = exp_subtype(e);
1140 name = exp_name(e);
1141 if (!name && e->type == e_column && e->r)
1142 name = e->r;
1143 rname = exp_relname(e);
1144 if (!rname && e->type == e_column && e->l)
1145 rname = e->l;
1146
1147 if (mnstr_printf(out, "[ \"%s\",\t%u,\t%u,\t\"%s\",\t\"%s\",\t\"%s\"\t]\n", t->type->sqlname, t->digits, t->scale, schema ? schema : "", rname ? rname : "", name ? name : "") < 0) {
1148 return -1;
1149 }
1150 }
1151 }
1152 if (c->params) {
1153 int i;
1154
1155 q->paramlen = nparam;
1156 q->params = SA_NEW_ARRAY(q->sa, sql_subtype, nrows);
1157 for (n = c->params->h, i = 0; n; n = n->next, i++) {
1158 a = n->data;
1159 t = &a->type;
1160
1161 if (t) {
1162 if (mnstr_printf(out, "[ \"%s\",\t%u,\t%u,\tNULL,\tNULL,\tNULL\t]\n", t->type->sqlname, t->digits, t->scale) < 0) {
1163 return -1;
1164 }
1165 /* add to the query cache parameters */
1166 q->params[i] = *t;
1167 } else {
1168 return -1;
1169 }
1170 }
1171 }
1172 if (mvc_export_warning(out, w) != 1)
1173 return -1;
1174 return 0;
1175}
1176
1177
1178/*
1179 * improved formatting of positive integers
1180 */
1181
1182static int
1183mvc_send_bte(stream *s, bte cnt)
1184{
1185 char buf[50], *b;
1186 int neg = cnt < 0;
1187 if (neg)
1188 cnt = -cnt;
1189 b = buf + 49;
1190 do {
1191 *b-- = (char) ('0' + (cnt % 10));
1192 cnt /= 10;
1193 } while (cnt > 0);
1194 if (neg)
1195 *b = '-';
1196 else
1197 b++;
1198 return mnstr_write(s, b, 50 - (b - buf), 1) == 1;
1199}
1200
1201static int
1202mvc_send_sht(stream *s, sht cnt)
1203{
1204 char buf[50], *b;
1205 int neg = cnt < 0;
1206 if (neg)
1207 cnt = -cnt;
1208 b = buf + 49;
1209 do {
1210 *b-- = (char) ('0' + (cnt % 10));
1211 cnt /= 10;
1212 } while (cnt > 0);
1213 if (neg)
1214 *b = '-';
1215 else
1216 b++;
1217 return mnstr_write(s, b, 50 - (b - buf), 1) == 1;
1218}
1219
1220static int
1221mvc_send_int(stream *s, int cnt)
1222{
1223 char buf[50], *b;
1224 int neg = cnt < 0;
1225 if (neg)
1226 cnt = -cnt;
1227 b = buf + 49;
1228 do {
1229 *b-- = (char) ('0' + (cnt % 10));
1230 cnt /= 10;
1231 } while (cnt > 0);
1232 if (neg)
1233 *b = '-';
1234 else
1235 b++;
1236 return mnstr_write(s, b, 50 - (b - buf), 1) == 1;
1237}
1238
1239static int
1240mvc_send_lng(stream *s, lng cnt)
1241{
1242 char buf[50], *b;
1243 int neg = cnt < 0;
1244 if (neg)
1245 cnt = -cnt;
1246 b = buf + 49;
1247 do {
1248 *b-- = (char) ('0' + (cnt % 10));
1249 cnt /= 10;
1250 } while (cnt > 0);
1251 if (neg)
1252 *b = '-';
1253 else
1254 b++;
1255 return mnstr_write(s, b, 50 - (b - buf), 1) == 1;
1256}
1257
1258#ifdef HAVE_HGE
1259static int
1260mvc_send_hge(stream *s, hge cnt){
1261 char buf[50], *b;
1262 int neg = cnt <0;
1263 if(neg) cnt = -cnt;
1264 b= buf+49;
1265 do{
1266 *b--= (char) ('0'+ (cnt % 10));
1267 cnt /=10;
1268 } while(cnt>0);
1269 if( neg)
1270 *b = '-';
1271 else b++;
1272 return mnstr_write(s, b, 50- (b-buf),1)==1;
1273}
1274#endif
1275
1276int
1277convert2str(mvc *m, sql_class eclass, int d, int sc, int has_tz, ptr p, int mtype, char **buf, int len)
1278{
1279 size_t len2 = (size_t) len;
1280 ssize_t l = 0;
1281
1282 if (!p || ATOMcmp(mtype, ATOMnilptr(mtype), p) == 0) {
1283 (*buf)[0] = '\200';
1284 (*buf)[1] = 0;
1285 } else if (eclass == EC_DEC) {
1286 l = dec_tostr((void *) (ptrdiff_t) sc, buf, &len2, mtype, p);
1287 } else if (eclass == EC_TIME) {
1288 struct time_res ts_res;
1289 ts_res.has_tz = has_tz;
1290 ts_res.fraction = d ? d - 1 : 0;
1291 ts_res.timezone = m->timezone;
1292 l = sql_time_tostr((void *) &ts_res, buf, &len2, mtype, p);
1293
1294 } else if (eclass == EC_TIMESTAMP) {
1295 struct time_res ts_res;
1296 ts_res.has_tz = has_tz;
1297 ts_res.fraction = d ? d - 1 : 0;
1298 ts_res.timezone = m->timezone;
1299 l = sql_timestamp_tostr((void *) &ts_res, buf, &len2, mtype, p);
1300 } else if (eclass == EC_BIT) {
1301 bit b = *(bit *) p;
1302 if (len <= 0 || len > 5) {
1303 if (b)
1304 strcpy(*buf, "true");
1305 else
1306 strcpy(*buf, "false");
1307 } else {
1308 (*buf)[0] = b?'t':'f';
1309 (*buf)[1] = 0;
1310 }
1311 } else {
1312 l = (*BATatoms[mtype].atomToStr) (buf, &len2, p, false);
1313 }
1314 return (int) l;
1315}
1316
1317static int
1318export_value(mvc *m, stream *s, sql_class eclass, const char *sqlname, int d, int sc, ptr p, int mtype, char **buf, size_t *len, const char *ns)
1319{
1320 int ok = 0;
1321 ssize_t l = 0;
1322
1323 if (!p || ATOMcmp(mtype, ATOMnilptr(mtype), p) == 0) {
1324 size_t ll = strlen(ns);
1325 ok = (mnstr_write(s, ns, ll, 1) == 1);
1326 } else if (eclass == EC_DEC) {
1327 l = dec_tostr((void *) (ptrdiff_t) sc, buf, len, mtype, p);
1328 if (l > 0)
1329 ok = (mnstr_write(s, *buf, l, 1) == 1);
1330 } else if (eclass == EC_TIME) {
1331 struct time_res ts_res;
1332 ts_res.has_tz = (strcmp(sqlname, "timetz") == 0);
1333 ts_res.fraction = d ? d - 1 : 0;
1334 ts_res.timezone = m->timezone;
1335 l = sql_time_tostr((void *) &ts_res, buf, len, mtype, p);
1336 if (l >= 0)
1337 ok = (mnstr_write(s, *buf, l, 1) == 1);
1338 } else if (eclass == EC_TIMESTAMP) {
1339 struct time_res ts_res;
1340 ts_res.has_tz = (strcmp(sqlname, "timestamptz") == 0);
1341 ts_res.fraction = d ? d - 1 : 0;
1342 ts_res.timezone = m->timezone;
1343 l = sql_timestamp_tostr((void *) &ts_res, buf, len, mtype, p);
1344 if (l >= 0)
1345 ok = (mnstr_write(s, *buf, l, 1) == 1);
1346 } else if (eclass == EC_SEC) {
1347 l = dec_tostr((void *) (ptrdiff_t) 3, buf, len, mtype, p);
1348 if (l >= 0)
1349 ok = mnstr_write(s, *buf, l, 1) == 1;
1350 } else {
1351 switch (mtype) {
1352 case TYPE_bte:
1353 ok = mvc_send_bte(s, *(bte *) p);
1354 break;
1355 case TYPE_sht:
1356 ok = mvc_send_sht(s, *(sht *) p);
1357 break;
1358 case TYPE_int:
1359 ok = mvc_send_int(s, *(int *) p);
1360 break;
1361 case TYPE_lng:
1362 ok = mvc_send_lng(s, *(lng *) p);
1363 break;
1364#ifdef HAVE_HGE
1365 case TYPE_hge:
1366 ok = mvc_send_hge(s, *(hge*)p);
1367 break;
1368#endif
1369 default:
1370 l = (*BATatoms[mtype].atomToStr) (buf, len, p, true);
1371 if (l >= 0)
1372 ok = (mnstr_write(s, *buf, l, 1) == 1);
1373 }
1374 }
1375 return ok;
1376}
1377
1378static int
1379mvc_export_row(backend *b, stream *s, res_table *t, const char *btag, const char *sep, const char *rsep, const char *ssep, const char *ns)
1380{
1381 mvc *m = b->mvc;
1382 size_t seplen = strlen(sep);
1383 size_t rseplen = strlen(rsep);
1384 char *buf = NULL;
1385 size_t len = 0;
1386 int i, ok = 1;
1387 int csv = (b->output_format == OFMT_CSV);
1388 int json = (b->output_format == OFMT_JSON);
1389 if (!s)
1390 return 0;
1391
1392 (void) ssep;
1393 if (csv && btag[0])
1394 ok = (mnstr_write(s, btag, strlen(btag), 1) == 1);
1395 if (json) {
1396 sep = ", ";
1397 seplen = strlen(sep);
1398 }
1399 for (i = 0; i < t->nr_cols && ok; i++) {
1400 res_col *c = t->cols + i;
1401
1402 if (i != 0) {
1403 ok = (mnstr_write(s, sep, seplen, 1) == 1);
1404 if (!ok)
1405 break;
1406 }
1407 if (json) {
1408 mnstr_write(s, c->name, strlen(c->name), 1);
1409 mnstr_write(s, ": ", 2, 1);
1410 }
1411 ok = export_value(m, s, c->type.type->eclass, c->type.type->sqlname, c->type.digits, c->type.scale, c->p, c->mtype, &buf, &len, ns);
1412 }
1413 _DELETE(buf);
1414 if (ok)
1415 ok = (mnstr_write(s, rsep, rseplen, 1) == 1);
1416 m->results = res_tables_remove(m->results, t);
1417 return (ok) ? 0 : -1;
1418}
1419
1420static int type_supports_binary_transfer(sql_type *type) {
1421 return
1422 type->eclass == EC_BIT ||
1423 type->eclass == EC_POS ||
1424 type->eclass == EC_CHAR ||
1425 type->eclass == EC_STRING ||
1426 type->eclass == EC_DEC ||
1427 type->eclass == EC_BLOB ||
1428 type->eclass == EC_FLT ||
1429 type->eclass == EC_NUM ||
1430 type->eclass == EC_DATE ||
1431 type->eclass == EC_TIME ||
1432 type->eclass == EC_SEC ||
1433 type->eclass == EC_MONTH ||
1434 type->eclass == EC_TIMESTAMP;
1435}
1436
1437static int write_str_term(stream* s, const char* const val) {
1438 return mnstr_writeStr(s, val) && mnstr_writeBte(s, 0);
1439}
1440
1441// align to 8 bytes
1442static char*
1443eight_byte_align(char* ptr) {
1444 return (char*) (((size_t) ptr + 7) & ~7);
1445}
1446
1447static int
1448mvc_export_table_prot10(backend *b, stream *s, res_table *t, BAT *order, BUN offset, BUN nr) {
1449 lng count = 0;
1450 size_t row = 0;
1451 size_t srow = 0;
1452 size_t varsized = 0;
1453 size_t length_prefixed = 0;
1454 lng fixed_lengths = 0;
1455 int fres = 0;
1456 size_t i = 0;
1457 size_t bsize = b->client->blocksize;
1458 BATiter *iterators = NULL;
1459 char *result = NULL;
1460 size_t length = 0;
1461 int initial_transfer = 1;
1462
1463 (void) order; // FIXME: respect explicitly ordered output
1464
1465 iterators = GDKzalloc(sizeof(BATiter) * t->nr_cols);
1466 if (!iterators) {
1467 return -1;
1468 }
1469
1470 // ensure the buffer is currently empty
1471 assert(bs2_buffer(s).pos == 0);
1472
1473 // inspect all the columns to figure out how many bytes it takes to transfer one row
1474 for (i = 0; i < (size_t) t->nr_cols; i++) {
1475 res_col *c = t->cols + i;
1476 BAT *b = BATdescriptor(c->b);
1477 int mtype;
1478 size_t typelen;
1479 int convert_to_string = !type_supports_binary_transfer(c->type.type);
1480 sql_type *type = c->type.type;
1481
1482 if (b == NULL) {
1483 while (i > 0) {
1484 i--;
1485 BBPunfix(iterators[i].b->batCacheid);
1486 }
1487 GDKfree(iterators);
1488 return -1;
1489 }
1490 mtype = b->ttype;
1491 typelen = ATOMsize(mtype);
1492 iterators[i] = bat_iterator(b);
1493
1494 if (type->eclass == EC_TIMESTAMP || type->eclass == EC_DATE) {
1495 // dates and timestamps are converted to Unix Timestamps
1496 mtype = TYPE_lng;
1497 typelen = sizeof(lng);
1498 }
1499 if (ATOMvarsized(mtype) || convert_to_string) {
1500 varsized++;
1501 length_prefixed++;
1502 } else {
1503 fixed_lengths += typelen;
1504 }
1505 }
1506
1507 // now perform the actual transfer
1508 row = srow = offset;
1509 count = offset + nr;
1510 while (row < (size_t) count) {
1511 char* message_header;
1512 char *buf = bs2_buffer(s).buf;
1513 size_t crow = 0;
1514 size_t bytes_left = bsize - sizeof(lng) - 2 * sizeof(char) - 1;
1515 // potential padding that has to be added for each column
1516 bytes_left -= t->nr_cols * 7;
1517
1518 // every varsized member has an 8-byte header indicating the length of the header in the block
1519 // subtract this from the amount of bytes left
1520 bytes_left -= length_prefixed * sizeof(lng);
1521
1522 if (varsized == 0) {
1523 // no varsized elements, so we can immediately compute the amount of elements
1524 if (fixed_lengths == 0) {
1525 row = (size_t) count;
1526 } else {
1527 row = (size_t) (srow + bytes_left / fixed_lengths);
1528 row = row > (size_t) count ? (size_t) count : row;
1529 }
1530 } else {
1531 size_t rowsize = 0;
1532 // we have varsized elements, so we have to loop to determine how many rows fit into a buffer
1533 while (row < (size_t) count) {
1534 rowsize = (size_t) fixed_lengths;
1535 for (i = 0; i < (size_t) t->nr_cols; i++) {
1536 res_col *c = t->cols + i;
1537 int mtype = iterators[i].b->ttype;
1538 int convert_to_string = !type_supports_binary_transfer(c->type.type);
1539 if (convert_to_string || ATOMvarsized(mtype)) {
1540 if (c->type.type->eclass == EC_BLOB) {
1541 blob *b = (blob*) BUNtvar(iterators[i], row);
1542 rowsize += sizeof(lng) + ((b->nitems == ~(size_t) 0) ? 0 : b->nitems);
1543 } else {
1544 ssize_t slen = 0;
1545 if (convert_to_string) {
1546 void *element = (void*) BUNtail(iterators[i], crow);
1547 if ((slen = BATatoms[mtype].atomToStr(&result, &length, element, false)) < 0) {
1548 fres = -1;
1549 goto cleanup;
1550 }
1551 } else {
1552 slen = (ssize_t) strlen((const char*) BUNtvar(iterators[i], row));
1553 }
1554 rowsize += slen + 1;
1555 }
1556 }
1557 }
1558 if (bytes_left < rowsize) {
1559 break;
1560 }
1561 bytes_left -= rowsize;
1562 row++;
1563 }
1564 if (row == srow) {
1565 lng new_size = rowsize + 1024;
1566 if (!mnstr_writeLng(s, (lng) -1) ||
1567 !mnstr_writeLng(s, new_size) ||
1568 mnstr_flush(s) < 0) {
1569 fres = -1;
1570 goto cleanup;
1571 }
1572 row = srow + 1;
1573 if (bs2_resizebuf(s, (size_t) new_size) < 0) {
1574 // failed to resize stream buffer
1575 fres = -1;
1576 goto cleanup;
1577 }
1578 buf = bs2_buffer(s).buf;
1579 bsize = (size_t) new_size;
1580 }
1581 }
1582
1583 // have to transfer at least one row
1584 assert(row > srow);
1585 // buffer has to be empty currently
1586 assert(bs2_buffer(s).pos == 0);
1587
1588 // initial message
1589 message_header = "+\n";
1590 if (initial_transfer == 0) {
1591 // continuation message
1592 message_header = "-\n";
1593 }
1594 initial_transfer = 0;
1595
1596 if (!mnstr_writeStr(s, message_header) || !mnstr_writeLng(s, (lng)(row - srow))) {
1597 fres = -1;
1598 goto cleanup;
1599 }
1600 buf += sizeof(lng) + 2 * sizeof(char);
1601
1602 for (i = 0; i < (size_t) t->nr_cols; i++) {
1603 res_col *c = t->cols + i;
1604 int mtype = iterators[i].b->ttype;
1605 int convert_to_string = !type_supports_binary_transfer(c->type.type);
1606 buf = eight_byte_align(buf);
1607 if (ATOMvarsized(mtype) || convert_to_string) {
1608 if (c->type.type->eclass == EC_BLOB) {
1609 // transfer blobs as [lng][data] combination
1610 char *startbuf = buf;
1611 buf += sizeof(lng);
1612 for (crow = srow; crow < row; crow++) {
1613 blob *b = (blob*) BUNtvar(iterators[i], crow);
1614 if (b->nitems == ~(size_t) 0) {
1615 (*(lng*)buf) = mnstr_swap_lng(s, -1);
1616 buf += sizeof(lng);
1617 } else {
1618 (*(lng*)buf) = mnstr_swap_lng(s, (lng) b->nitems);
1619 buf += sizeof(lng);
1620 memcpy(buf, b->data, b->nitems);
1621 buf += b->nitems;
1622 }
1623 }
1624 // after the loop we know the size of the column, so write it
1625 *((lng*)startbuf) = mnstr_swap_lng(s, buf - (startbuf + sizeof(lng)));
1626 } else {
1627 // for variable length strings and large fixed strings we use varints
1628 // variable columns are prefixed by a length,
1629 // but since we don't know the length yet, just skip over it for now
1630 char *startbuf = buf;
1631 buf += sizeof(lng);
1632 for (crow = srow; crow < row; crow++) {
1633 void *element = (void*) BUNtail(iterators[i], crow);
1634 const char* str;
1635 if (convert_to_string) {
1636 if (BATatoms[mtype].atomCmp(element, BATatoms[mtype].atomNull) == 0) {
1637 str = str_nil;
1638 } else {
1639 if (BATatoms[mtype].atomToStr(&result, &length, element, false) < 0) {
1640 fres = -1;
1641 goto cleanup;
1642 }
1643 str = result;
1644 }
1645 } else {
1646 str = (char*) element;
1647 }
1648 buf = stpcpy(buf, str) + 1;
1649 assert(buf - bs2_buffer(s).buf <= (lng) bsize);
1650 }
1651 *((lng*)startbuf) = mnstr_swap_lng(s, buf - (startbuf + sizeof(lng)));
1652 }
1653 } else {
1654 size_t atom_size = ATOMsize(mtype);
1655 if (c->type.type->eclass == EC_DEC) {
1656 atom_size = ATOMsize(mtype);
1657 }
1658 if (c->type.type->eclass == EC_TIMESTAMP) {
1659 // convert timestamp values to epoch
1660 lng time;
1661 size_t j = 0;
1662 bool swap = mnstr_get_swapbytes(s);
1663 timestamp *times = (timestamp*) Tloc(iterators[i].b, srow);
1664 timestamp epoch = timestamp_create(date_create(1970, 1, 1), daytime_create(0, 0, 0, 0));
1665 lng *bufptr = (lng*) buf;
1666 for(j = 0; j < (row - srow); j++) {
1667 time = timestamp_diff(times[j], epoch) / 1000;
1668 bufptr[j] = swap ? long_long_SWAP(time) : time;
1669 }
1670 atom_size = sizeof(lng);
1671 } else if (c->type.type->eclass == EC_DATE) {
1672 // convert dates into timestamps since epoch
1673 lng time;
1674 size_t j = 0;
1675 bool swap = mnstr_get_swapbytes(s);
1676 date *dates = (date*) Tloc(iterators[i].b, srow);
1677 date epoch = date_create(1970, 1, 1);
1678 lng *bufptr = (lng*) buf;
1679 for(j = 0; j < (row - srow); j++) {
1680 time = date_diff(dates[j], epoch) * 24*60*60*LL_CONSTANT(1000);
1681 bufptr[j] = swap ? long_long_SWAP(time) : time;
1682 }
1683 atom_size = sizeof(lng);
1684 } else {
1685 if (mnstr_get_swapbytes(s)) {
1686 size_t j = 0;
1687 switch (ATOMstorage(mtype)) {
1688 case TYPE_sht: {
1689 short *bufptr = (short*) buf;
1690 short *exported_values = (short*) Tloc(iterators[i].b, srow);
1691 for(j = 0; j < (row - srow); j++) {
1692 bufptr[j] = short_int_SWAP(exported_values[j]);
1693 }
1694 break;
1695 }
1696 case TYPE_int: {
1697 int *bufptr = (int*) buf;
1698 int *exported_values = (int*) Tloc(iterators[i].b, srow);
1699 for(j = 0; j < (row - srow); j++) {
1700 bufptr[j] = normal_int_SWAP(exported_values[j]);
1701 }
1702 break;
1703 }
1704 case TYPE_lng: {
1705 lng *bufptr = (lng*) buf;
1706 lng *exported_values = (lng*) Tloc(iterators[i].b, srow);
1707 for(j = 0; j < (row - srow); j++) {
1708 bufptr[j] = long_long_SWAP(exported_values[j]);
1709 }
1710 break;
1711 }
1712#ifdef HAVE_HGE
1713 case TYPE_hge: {
1714 hge *bufptr = (hge*) buf;
1715 hge *exported_values = (hge*) Tloc(iterators[i].b, srow);
1716 for(j = 0; j < (row - srow); j++) {
1717 bufptr[j] = huge_int_SWAP(exported_values[j]);
1718 }
1719 break;
1720 }
1721#endif
1722 }
1723 } else {
1724 memcpy(buf, Tloc(iterators[i].b, srow), (row - srow) * atom_size);
1725 }
1726 }
1727 buf += (row - srow) * atom_size;
1728 }
1729 }
1730
1731 assert(buf >= bs2_buffer(s).buf);
1732 if (buf - bs2_buffer(s).buf > (lng) bsize) {
1733 fprintf(stderr, "Too many bytes in the buffer.\n");
1734 fres = -1;
1735 goto cleanup;
1736 }
1737
1738 bs2_setpos(s, buf - bs2_buffer(s).buf);
1739 // flush the current chunk
1740 if (mnstr_flush(s) < 0) {
1741 fres = -1;
1742 goto cleanup;
1743 }
1744 srow = row;
1745 }
1746cleanup:
1747 if (iterators) {
1748 for (i = 0; i < (size_t) t->nr_cols; i++)
1749 BBPunfix(iterators[i].b->batCacheid);
1750 GDKfree(iterators);
1751 }
1752 if (result) {
1753 GDKfree(result);
1754 }
1755 if (mnstr_errnr(s))
1756 return -1;
1757 return fres;
1758}
1759
1760static int
1761mvc_export_table(backend *b, stream *s, res_table *t, BAT *order, BUN offset, BUN nr, const char *btag, const char *sep, const char *rsep, const char *ssep, const char *ns)
1762{
1763 mvc *m = b->mvc;
1764 Tablet as;
1765 Column *fmt;
1766 int i;
1767 struct time_res *tres;
1768 int csv = (b->output_format == OFMT_CSV);
1769 int json = (b->output_format == OFMT_JSON);
1770 char *bj;
1771
1772 if (!t)
1773 return -1;
1774 if (!s)
1775 return 0;
1776
1777 if (b->client->protocol == PROTOCOL_10) {
1778 return mvc_export_table_prot10(b, s, t, order, offset, nr);
1779 }
1780
1781 as.nr_attrs = t->nr_cols + 1; /* for the leader */
1782 as.nr = nr;
1783 as.offset = offset;
1784 fmt = as.format = (Column *) GDKzalloc(sizeof(Column) * (as.nr_attrs + 1));
1785 tres = GDKzalloc(sizeof(struct time_res) * (as.nr_attrs));
1786 if(fmt == NULL || tres == NULL) {
1787 GDKfree(fmt);
1788 GDKfree(tres);
1789 sql_error(m, 500, SQLSTATE(HY001) MAL_MALLOC_FAIL);
1790 return -1;
1791 }
1792
1793 fmt[0].c = NULL;
1794 fmt[0].sep = (csv) ? btag : "";
1795 fmt[0].rsep = rsep;
1796 fmt[0].seplen = _strlen(fmt[0].sep);
1797 fmt[0].ws = 0;
1798 fmt[0].nullstr = NULL;
1799
1800 for (i = 1; i <= t->nr_cols; i++) {
1801 res_col *c = t->cols + (i - 1);
1802
1803 if (!c->b)
1804 break;
1805
1806 fmt[i].c = BATdescriptor(c->b);
1807 if (fmt[i].c == NULL) {
1808 while (--i >= 1)
1809 BBPunfix(fmt[i].c->batCacheid);
1810 GDKfree(fmt);
1811 GDKfree(tres);
1812 return -1;
1813 }
1814 fmt[i].ci = bat_iterator(fmt[i].c);
1815 fmt[i].name = NULL;
1816 if (csv) {
1817 fmt[i].sep = ((i - 1) < (t->nr_cols - 1)) ? sep : rsep;
1818 fmt[i].seplen = _strlen(fmt[i].sep);
1819 fmt[i].rsep = rsep;
1820 }
1821 if (json) {
1822 res_col *p = t->cols + (i - 1);
1823
1824 /*
1825 * We define the "proper" way of returning
1826 * a relational table in json format as a
1827 * json array of objects, where each row is
1828 * represented as a json object.
1829 */
1830 if (i == 1) {
1831 bj = SA_NEW_ARRAY(m->sa, char, strlen(p->name) + strlen(btag));
1832 snprintf(bj, strlen(p->name) + strlen(btag), btag, p->name);
1833 fmt[i - 1].sep = bj;
1834 fmt[i - 1].seplen = _strlen(fmt[i - 1].sep);
1835 fmt[i - 1].rsep = NULL;
1836 } else if (i <= t->nr_cols) {
1837 bj = SA_NEW_ARRAY(m->sa, char, strlen(p->name) + strlen(sep));
1838 snprintf(bj, strlen(p->name) + 10, sep, p->name);
1839 fmt[i - 1].sep = bj;
1840 fmt[i - 1].seplen = _strlen(fmt[i - 1].sep);
1841 fmt[i - 1].rsep = NULL;
1842 }
1843 if (i == t->nr_cols) {
1844 fmt[i].sep = rsep;
1845 fmt[i].seplen = _strlen(fmt[i].sep);
1846 fmt[i].rsep = NULL;
1847 }
1848 }
1849 fmt[i].type = ATOMname(fmt[i].c->ttype);
1850 fmt[i].adt = fmt[i].c->ttype;
1851 fmt[i].tostr = &_ASCIIadt_toStr;
1852 fmt[i].frstr = &_ASCIIadt_frStr;
1853 fmt[i].extra = fmt + i;
1854 fmt[i].data = NULL;
1855 fmt[i].len = 0;
1856 fmt[i].ws = 0;
1857 fmt[i].quote = ssep ? ssep[0] : 0;
1858 fmt[i].nullstr = ns;
1859 if (c->type.type->eclass == EC_DEC) {
1860 fmt[i].tostr = &dec_tostr;
1861 fmt[i].frstr = &dec_frstr;
1862 fmt[i].extra = (void *) (ptrdiff_t) c->type.scale;
1863 } else if (c->type.type->eclass == EC_TIMESTAMP) {
1864 struct time_res *ts_res = tres + (i - 1);
1865 ts_res->has_tz = (strcmp(c->type.type->sqlname, "timestamptz") == 0);
1866 ts_res->fraction = c->type.digits ? c->type.digits - 1 : 0;
1867 ts_res->timezone = m->timezone;
1868
1869 fmt[i].tostr = &sql_timestamp_tostr;
1870 fmt[i].frstr = NULL;
1871 fmt[i].extra = ts_res;
1872 } else if (c->type.type->eclass == EC_TIME) {
1873 struct time_res *ts_res = tres + (i - 1);
1874 ts_res->has_tz = (strcmp(c->type.type->sqlname, "timetz") == 0);
1875 ts_res->fraction = c->type.digits ? c->type.digits - 1 : 0;
1876 ts_res->timezone = m->timezone;
1877
1878 fmt[i].tostr = &sql_time_tostr;
1879 fmt[i].frstr = NULL;
1880 fmt[i].extra = ts_res;
1881 } else if (c->type.type->eclass == EC_SEC) {
1882 fmt[i].tostr = &dec_tostr;
1883 fmt[i].frstr = &sec_frstr;
1884 fmt[i].extra = (void *) (ptrdiff_t) 3;
1885 } else {
1886 fmt[i].extra = fmt + i;
1887 }
1888 }
1889 if (i == t->nr_cols + 1) {
1890 TABLEToutput_file(&as, order, s);
1891 }
1892 for (i = 0; i <= t->nr_cols; i++) {
1893 fmt[i].sep = NULL;
1894 fmt[i].rsep = NULL;
1895 fmt[i].type = NULL;
1896 fmt[i].nullstr = NULL;
1897 }
1898 TABLETdestroy_format(&as);
1899 GDKfree(tres);
1900 if (mnstr_errnr(s))
1901 return -1;
1902 return 0;
1903}
1904
1905
1906static lng
1907get_print_width(int mtype, sql_class eclass, int digits, int scale, int tz, bat bid, ptr p)
1908{
1909 size_t count = 0, incr = 0;;
1910
1911 if (eclass == EC_SEC)
1912 incr = 1;
1913 else if (mtype == TYPE_oid)
1914 incr = 2;
1915 mtype = ATOMbasetype(mtype);
1916 if (mtype == TYPE_str) {
1917 if (eclass == EC_CHAR && digits) {
1918 return digits;
1919 } else {
1920 int l = 0;
1921 if (bid) {
1922 BAT *b = BATdescriptor(bid);
1923
1924 if (b) {
1925 /* in practice, b can be a
1926 * void(nil) bat, an oid bat
1927 * with all nil values, or an
1928 * empty void/oid bat */
1929 if (ATOMstorage(b->ttype) == TYPE_str)
1930 l = bat_max_strlength(b);
1931 else
1932 l = 0;
1933 BBPunfix(b->batCacheid);
1934 } else {
1935 assert(b);
1936 /* [Stefan.Manegold@cwi.nl]:
1937 * Instead of an assert() or simply ignoring the problem,
1938 * we could/should return an error code, but I don't know
1939 * which it the correct/suitable error code -1|0|1 ?
1940 *
1941 return -1|0|1 ;
1942 */
1943 }
1944 } else if (p) {
1945 l = STRwidth((const char *) p);
1946 if (is_int_nil(l))
1947 l = 0;
1948 }
1949 return l;
1950 }
1951 } else if (eclass == EC_NUM || eclass == EC_POS || eclass == EC_MONTH || eclass == EC_SEC) {
1952 count = 0;
1953 if (bid) {
1954 BAT *b = BATdescriptor(bid);
1955
1956 if (b) {
1957 if (mtype == TYPE_bte) {
1958 count = bat_max_btelength(b);
1959 } else if (mtype == TYPE_sht) {
1960 count = bat_max_shtlength(b);
1961 } else if (mtype == TYPE_int) {
1962 count = bat_max_intlength(b);
1963 } else if (mtype == TYPE_lng) {
1964 count = bat_max_lnglength(b);
1965#ifdef HAVE_HGE
1966 } else if (mtype == TYPE_hge) {
1967 count = bat_max_hgelength(b);
1968#endif
1969 } else if (mtype == TYPE_void) {
1970 count = 4;
1971 } else {
1972 assert(0);
1973 }
1974 count += incr;
1975 BBPunfix(b->batCacheid);
1976 } else {
1977 assert(b);
1978 /* [Stefan.Manegold@cwi.nl]:
1979 * Instead of an assert() or simply ignoring the problem,
1980 * we could/should return an error code, but I don't know
1981 * which it the correct/suitable error code -1|0|1 ?
1982 *
1983 return -1|0|1 ;
1984 */
1985 }
1986 } else {
1987 if (p) {
1988#ifdef HAVE_HGE
1989 hge val = 0;
1990#else
1991 lng val = 0;
1992#endif
1993 if (mtype == TYPE_bte) {
1994 val = *((bte *) p);
1995 } else if (mtype == TYPE_sht) {
1996 val = *((sht *) p);
1997 } else if (mtype == TYPE_int) {
1998 val = *((int *) p);
1999 } else if (mtype == TYPE_lng) {
2000 val = *((lng *) p);
2001#ifdef HAVE_HGE
2002 } else if (mtype == TYPE_hge) {
2003 val = *((hge *) p);
2004#endif
2005 } else {
2006 assert(0);
2007 }
2008
2009 if (val < 0)
2010 count++;
2011 while (val /= 10)
2012 count++;
2013 count++;
2014 count += incr;
2015 } else {
2016 count = 0;
2017 }
2018 }
2019 if (eclass == EC_SEC && count < 5)
2020 count = 5;
2021 return count;
2022 /* the following two could be done once by taking the
2023 max value and calculating the number of digits from that
2024 value, instead of the maximum values taken now, which
2025 include the optional sign */
2026 } else if (eclass == EC_FLT) {
2027 /* floats are printed using "%.9g":
2028 * [sign]+digit+period+[max 8 digits]+E+[sign]+[max 2 digits] */
2029 if (mtype == TYPE_flt) {
2030 return 15;
2031 /* doubles are printed using "%.17g":
2032 * [sign]+digit+period+[max 16 digits]+E+[sign]+[max 3 digits] */
2033 } else { /* TYPE_dbl */
2034 return 24;
2035 }
2036 } else if (eclass == EC_DEC) {
2037 count = 1 + digits;
2038 if (scale > 0)
2039 count += 1;
2040 return count;
2041 } else if (eclass == EC_DATE) {
2042 return 10;
2043 } else if (eclass == EC_TIME) {
2044 count = 8;
2045 if (tz) /* time zone */
2046 count += 6; /* +03:30 */
2047 if (digits > 1) /* fractional seconds precision (including dot) */
2048 count += digits;
2049 return count;
2050 } else if (eclass == EC_TIMESTAMP) {
2051 count = 10 + 1 + 8;
2052 if (tz) /* time zone */
2053 count += 6; /* +03:30 */
2054 if (digits) /* fractional seconds precision */
2055 count += digits;
2056 return count;
2057 } else if (eclass == EC_BIT) {
2058 return 5; /* max(strlen("true"), strlen("false")) */
2059 } else if (strcmp(ATOMname(mtype), "uuid") == 0) {
2060 return 36; /* xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx */
2061 } else {
2062 return 0;
2063 }
2064}
2065
2066static int
2067export_length(stream *s, int mtype, sql_class eclass, int digits, int scale, int tz, bat bid, ptr p)
2068{
2069 int ok = 1;
2070 lng length = get_print_width(mtype, eclass, digits, scale, tz, bid, p);
2071 ok = mvc_send_lng(s, length);
2072 return ok;
2073}
2074
2075int
2076mvc_export_operation(backend *b, stream *s, str w, lng starttime, lng mal_optimizer)
2077{
2078 mvc *m = b->mvc;
2079
2080 assert(m->type == Q_SCHEMA || m->type == Q_TRANS);
2081 if (m->type == Q_SCHEMA) {
2082 if (!s || mnstr_printf(s, "&3 " LLFMT " " LLFMT "\n", starttime > 0 ? GDKusec() - starttime : 0, mal_optimizer) < 0)
2083 return -1;
2084 } else {
2085 if (m->session->auto_commit) {
2086 if (mnstr_write(s, "&4 t\n", 5, 1) != 1)
2087 return -1;
2088 } else {
2089 if (mnstr_write(s, "&4 f\n", 5, 1) != 1)
2090 return -1;
2091 }
2092 }
2093
2094 if (mvc_export_warning(s, w) != 1)
2095 return -1;
2096 return 0;
2097}
2098
2099int
2100mvc_export_affrows(backend *b, stream *s, lng val, str w, oid query_id, lng starttime, lng maloptimizer)
2101{
2102 mvc *m = b->mvc;
2103 /* if we don't have a stream, nothing can go wrong, so we return
2104 * success. This is especially vital for execution of internal SQL
2105 * commands, since they don't get a stream to suppress their output.
2106 * If we would fail on having no stream here, those internal commands
2107 * fail too.
2108 */
2109 if (!s)
2110 return 0;
2111
2112 m->rowcnt = val;
2113 stack_set_number(m, "rowcnt", m->rowcnt);
2114 if (mnstr_write(s, "&2 ", 3, 1) != 1 ||
2115 !mvc_send_lng(s, val) ||
2116 mnstr_write(s, " ", 1, 1) != 1 ||
2117 !mvc_send_lng(s, m->last_id) ||
2118 mnstr_write(s, " ", 1, 1) != 1 ||
2119 !mvc_send_lng(s, (lng) query_id) ||
2120 mnstr_write(s, " ", 1, 1) != 1 ||
2121 !mvc_send_lng(s, starttime > 0 ? GDKusec() - starttime : 0) ||
2122 mnstr_write(s, " ", 1, 1) != 1 ||
2123 !mvc_send_lng(s, maloptimizer) ||
2124 mnstr_write(s, " ", 1, 1) != 1 ||
2125 !mvc_send_lng(s, m->Topt) ||
2126 mnstr_write(s, "\n", 1, 1) != 1)
2127 return -1;
2128 if (mvc_export_warning(s, w) != 1)
2129 return -1;
2130
2131 return 0;
2132}
2133
2134static int
2135export_error(BAT *order)
2136{
2137 if (order)
2138 BBPunfix(order->batCacheid);
2139 return -1;
2140}
2141
2142static int
2143mvc_export_head_prot10(backend *b, stream *s, int res_id, int only_header, int compute_lengths) {
2144 mvc *m = b->mvc;
2145 size_t i = 0;
2146 BUN count = 0;
2147 res_table *t = res_tables_find(m->results, res_id);
2148 BAT *order = NULL;
2149 int fres = 0;
2150
2151 if (!t || !s) {
2152 return 0;
2153 }
2154
2155 /* tuple count */
2156 if (only_header) {
2157 if (t->order) {
2158 order = BBPquickdesc(t->order, false);
2159 if (!order)
2160 return -1;
2161
2162 count = BATcount(order);
2163 } else
2164 count = 1;
2165 }
2166 m->rowcnt = count;
2167
2168 // protocol 10 result sets start with "*\n" followed by the binary data:
2169 // [tableid][queryid][rowcount][colcount][timezone]
2170 if (!mnstr_writeStr(s, "*\n") ||
2171 !mnstr_writeInt(s, t->id) ||
2172 !mnstr_writeLng(s, (lng) t->query_id) ||
2173 !mnstr_writeLng(s, count) || !mnstr_writeLng(s, (lng) t->nr_cols)) {
2174 fres = -1;
2175 goto cleanup;
2176 }
2177 // write timezone to the client
2178 if (!mnstr_writeInt(s, m->timezone)) {
2179 fres = -1;
2180 goto cleanup;
2181 }
2182
2183 // after that, the data of the individual columns is written
2184 for (i = 0; i < (size_t) t->nr_cols; i++) {
2185 res_col *c = t->cols + i;
2186 BAT *b = BATdescriptor(c->b);
2187 int mtype;
2188 int typelen;
2189 int nil_len = -1;
2190 int nil_type;
2191 int retval = -1;
2192 int convert_to_string = !type_supports_binary_transfer(c->type.type);
2193 sql_type *type = c->type.type;
2194 lng print_width = -1;
2195
2196 if (b == NULL)
2197 return -1;
2198
2199 mtype = b->ttype;
2200 typelen = ATOMsize(mtype);
2201 nil_type = ATOMstorage(mtype);
2202
2203 // if the client wants print widths, we compute them for this column
2204 if (compute_lengths) {
2205 print_width = get_print_width(mtype, type->eclass, c->type.digits, c->type.scale, type_has_tz(&c->type), b->batCacheid, c->p);
2206 }
2207
2208 if (type->eclass == EC_TIMESTAMP || type->eclass == EC_DATE) {
2209 // timestamps are converted to Unix Timestamps
2210 mtype = TYPE_lng;
2211 typelen = sizeof(lng);
2212 }
2213
2214 if (convert_to_string) {
2215 nil_type = TYPE_str;
2216 }
2217
2218 if (ATOMvarsized(mtype) || convert_to_string) {
2219 // variable length columns have typelen set to -1
2220 typelen = -1;
2221 nil_len = (int) strlen(str_nil) + 1;
2222 } else {
2223 nil_len = typelen;
2224 }
2225
2226 // column data has the following binary format:
2227 // [tablename]\0[columnname]\0[sqltypename]\0[typelen][digits][scale][nil_length][nil_value][print_width]
2228 if (!write_str_term(s, c->tn) || !write_str_term(s, c->name) || !write_str_term(s, type->sqlname) ||
2229 !mnstr_writeInt(s, typelen) || !mnstr_writeInt(s, c->type.digits) || !mnstr_writeInt(s, type->eclass == EC_SEC ? 3 : c->type.scale)) {
2230 fres = -1;
2231 BBPunfix(b->batCacheid);
2232 goto cleanup;
2233 }
2234
2235 if ((!b->tnil && b->tnonil) || type->eclass == EC_BLOB) {
2236 nil_len = 0;
2237 }
2238
2239 BBPunfix(b->batCacheid);
2240
2241 // write NULL values for this column to the stream
2242 // NULL values are encoded as [size:int][NULL value] ([size] is always [typelen] for fixed size columns)
2243 if (!mnstr_writeInt(s, nil_len)) {
2244 fres = -1;
2245 goto cleanup;
2246 }
2247 // transfer the actual NULL value
2248 if (nil_len > 0) {
2249 switch(nil_type) {
2250 case TYPE_str:
2251 retval = write_str_term(s, str_nil);
2252 break;
2253 case TYPE_bit:
2254 case TYPE_bte:
2255 retval = mnstr_writeBte(s, bte_nil);
2256 break;
2257 case TYPE_sht:
2258 retval = mnstr_writeSht(s, sht_nil);
2259 break;
2260 case TYPE_int:
2261 retval = mnstr_writeInt(s, int_nil);
2262 break;
2263 case TYPE_lng:
2264 retval = mnstr_writeLng(s, lng_nil);
2265 break;
2266 case TYPE_flt:
2267 retval = mnstr_writeFlt(s, flt_nil);
2268 break;
2269 case TYPE_dbl:
2270 retval = mnstr_writeDbl(s, dbl_nil);
2271 break;
2272#ifdef HAVE_HGE
2273 case TYPE_hge:
2274 retval = mnstr_writeHge(s, hge_nil);
2275 break;
2276#endif
2277 case TYPE_void:
2278 break;
2279 default:
2280 assert(0);
2281 fres = -1;
2282 goto cleanup;
2283 }
2284 }
2285 if (!retval) {
2286 fres = -1;
2287 goto cleanup;
2288 }
2289 // transfer the computed print width
2290 if (!mnstr_writeLng(s, print_width)) {
2291 fres = -1;
2292 goto cleanup;
2293 }
2294 }
2295 if (mnstr_flush(s) < 0) {
2296 fres = -1;
2297 goto cleanup;
2298 }
2299cleanup:
2300 return fres;
2301}
2302
2303int
2304mvc_export_head(backend *b, stream *s, int res_id, int only_header, int compute_lengths, lng starttime, lng maloptimizer)
2305{
2306 mvc *m = b->mvc;
2307 int i, res = 0;
2308 BUN count = 0;
2309 res_table *t = res_tables_find(m->results, res_id);
2310 BAT *order = NULL;
2311
2312 if (!s || !t)
2313 return 0;
2314
2315
2316 if (b->client->protocol == PROTOCOL_10) {
2317 // export head result set 10
2318 return mvc_export_head_prot10(b, s, res_id, only_header, compute_lengths);
2319 }
2320
2321 /* query type: Q_TABLE */
2322 if (!(mnstr_write(s, "&1 ", 3, 1) == 1))
2323 return -1;
2324
2325 /* id */
2326 if (!mvc_send_int(s, t->id) || mnstr_write(s, " ", 1, 1) != 1)
2327 return -1;
2328
2329 /* tuple count */
2330 if (only_header) {
2331 if (t->order) {
2332 order = BBPquickdesc(t->order, false);
2333 if (!order)
2334 return -1;
2335
2336 count = BATcount(order);
2337 } else
2338 count = 1;
2339 }
2340 m->rowcnt = count;
2341 stack_set_number(m, "rowcnt", m->rowcnt);
2342 if (!mvc_send_lng(s, (lng) count) || mnstr_write(s, " ", 1, 1) != 1)
2343 return -1;
2344
2345 /* column count */
2346 if (!mvc_send_int(s, t->nr_cols) || mnstr_write(s, " ", 1, 1) != 1)
2347 return -1;
2348
2349 /* row count, min(count, reply_size) */
2350 if (!mvc_send_int(s, (m->reply_size >= 0 && (BUN) m->reply_size < count) ? m->reply_size : (int) count))
2351 return -1;
2352
2353 // export query id
2354 if (mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, (lng) t->query_id))
2355 return -1;
2356
2357 // export query time
2358 if (mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, starttime > 0 ? GDKusec() - starttime : 0))
2359 return -1;
2360
2361 // export MAL optimizer time
2362 if (mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, maloptimizer))
2363 return -1;
2364
2365 if (mnstr_write(s, " ", 1, 1) != 1 || !mvc_send_lng(s, m->Topt))
2366 return -1;
2367
2368 if (mnstr_write(s, "\n% ", 3, 1) != 1)
2369 return -1;
2370 for (i = 0; i < t->nr_cols; i++) {
2371 res_col *c = t->cols + i;
2372 size_t len = strlen(c->tn);
2373
2374 if (len && mnstr_write(s, c->tn, len, 1) != 1)
2375 return -1;
2376 if (i + 1 < t->nr_cols && mnstr_write(s, ",\t", 2, 1) != 1)
2377 return -1;
2378 }
2379 if (mnstr_write(s, " # table_name\n% ", 16, 1) != 1)
2380 return -1;
2381
2382 for (i = 0; i < t->nr_cols; i++) {
2383 res_col *c = t->cols + i;
2384
2385 if (strpbrk(c->name, ", \t#\"\\")) {
2386 char *p;
2387 if (mnstr_write(s, "\"", 1, 1) != 1)
2388 return -1;
2389 for (p = c->name; *p; p++) {
2390 if (*p == '"' || *p == '\\') {
2391 if (mnstr_write(s, "\\", 1, 1) != 1)
2392 return -1;
2393 }
2394 if (mnstr_write(s, p, 1, 1) != 1)
2395 return -1;
2396 }
2397 if (mnstr_write(s, "\"", 1, 1) != 1)
2398 return -1;
2399 } else {
2400 if (mnstr_write(s, c->name, strlen(c->name), 1) != 1)
2401 return -1;
2402 }
2403
2404 if (i + 1 < t->nr_cols && mnstr_write(s, ",\t", 2, 1) != 1)
2405 return -1;
2406 }
2407 if (mnstr_write(s, " # name\n% ", 10, 1) != 1)
2408 return -1;
2409
2410 for (i = 0; i < t->nr_cols; i++) {
2411 res_col *c = t->cols + i;
2412
2413 if (mnstr_write(s, c->type.type->sqlname, strlen(c->type.type->sqlname), 1) != 1)
2414 return -1;
2415 if (i + 1 < t->nr_cols && mnstr_write(s, ",\t", 2, 1) != 1)
2416 return -1;
2417 }
2418 if (mnstr_write(s, " # type\n% ", 10, 1) != 1)
2419 return -1;
2420 if (compute_lengths) {
2421 for (i = 0; i < t->nr_cols; i++) {
2422 res_col *c = t->cols + i;
2423 int mtype = c->type.type->localtype;
2424 sql_class eclass = c->type.type->eclass;
2425
2426 if (!export_length(s, mtype, eclass, c->type.digits, c->type.scale, type_has_tz(&c->type), c->b, c->p))
2427 return -1;
2428 if (i + 1 < t->nr_cols && mnstr_write(s, ",\t", 2, 1) != 1)
2429 return -1;
2430 }
2431 if (mnstr_write(s, " # length\n", 10, 1) != 1)
2432 return -1;
2433 }
2434 if (m->sizeheader) {
2435 if (mnstr_write(s, "% ", 2, 1) != 1)
2436 return -1;
2437 for (i = 0; i < t->nr_cols; i++) {
2438 res_col *c = t->cols + i;
2439
2440 if (mnstr_printf(s, "%u %u", c->type.digits, c->type.scale) < 0)
2441 return -1;
2442 if (i + 1 < t->nr_cols && mnstr_write(s, ",\t", 2, 1) != 1)
2443 return -1;
2444 }
2445 if (mnstr_write(s, " # typesizes\n", 13, 1) != 1)
2446 return -1;
2447 }
2448 return res;
2449}
2450
2451static int
2452mvc_export_file(backend *b, stream *s, res_table *t)
2453{
2454 mvc *m = b->mvc;
2455 int res = 0;
2456 BUN count;
2457 BAT *order = NULL;
2458
2459 if (!t->order) {
2460 res = mvc_export_row(b, s, t, "", t->tsep, t->rsep, t->ssep, t->ns);
2461 } else {
2462 order = BATdescriptor(t->order);
2463 if (!order)
2464 return -1;
2465 count = BATcount(order);
2466
2467 res = mvc_export_table(b, s, t, order, 0, count, "", t->tsep, t->rsep, t->ssep, t->ns);
2468 BBPunfix(order->batCacheid);
2469 m->results = res_tables_remove(m->results, t);
2470 }
2471 return res;
2472}
2473
2474int
2475mvc_export_result(backend *b, stream *s, int res_id, bool header, lng starttime, lng maloptimizer)
2476{
2477 mvc *m = b->mvc;
2478 int clean = 0, res = 0;
2479 BUN count;
2480 res_table *t = res_tables_find(m->results, res_id);
2481 BAT *order = NULL;
2482 int json = (b->output_format == OFMT_JSON);
2483
2484 if (!s || !t)
2485 return 0;
2486
2487 /* Proudly supporting SQLstatementIntern's output flag */
2488 if (b->output_format == OFMT_NONE) {
2489 return 0;
2490 }
2491 /* we shouldn't have anything else but Q_TABLE here */
2492 assert(t->query_type == Q_TABLE);
2493 if (t->tsep) {
2494 if (header) {
2495 /* need header */
2496 mvc_export_head(b, s, t->id, TRUE, TRUE, starttime, maloptimizer);
2497 }
2498 return mvc_export_file(b, s, t);
2499 }
2500
2501 if (!json) {
2502 mvc_export_head(b, s, res_id, TRUE, TRUE, starttime, maloptimizer);
2503 }
2504
2505 assert(t->order);
2506
2507 order = BATdescriptor(t->order);
2508 if (!order)
2509 return -1;
2510
2511 count = m->reply_size;
2512 if (m->reply_size != -2 && (count <= 0 || count >= BATcount(order))) {
2513 count = BATcount(order);
2514 clean = 1;
2515 }
2516 if (json) {
2517 switch(count) {
2518 case 0:
2519 res = mvc_export_table(b, s, t, order, 0, count, "{\t", "", "}\n", "\"", "null");
2520 break;
2521 case 1:
2522 res = mvc_export_table(b, s, t, order, 0, count, "{\n\t\"%s\" : ", ",\n\t\"%s\" : ", "\n}\n", "\"", "null");
2523 break;
2524 case 2:
2525 res = mvc_export_table(b, s, t, order, 0, 1, "[\n\t{\n\t\t\"%s\" : ", ",\n\t\t\"%s\" : ", "\n\t},\n", "\"", "null");
2526 res = mvc_export_table(b, s, t, order, 1, count - 1, "\t{\n\t\t\"%s\" : ", ",\n\t\t\"%s\" : ", "\n\t}\n]\n", "\"", "null");
2527 break;
2528 default:
2529 res = mvc_export_table(b, s, t, order, 0, 1, "[\n\t{\n\t\t\"%s\" : ", ",\n\t\t\"%s\" : ", "\n\t},\n", "\"", "null");
2530 res = mvc_export_table(b, s, t, order, 1, count - 2, "\t{\n\t\t\"%s\" : ", ",\n\t\t\"%s\" : ", "\n\t},\n", "\"", "null");
2531 res = mvc_export_table(b, s, t, order, count - 1, 1, "\t{\n\t\t\"%s\" : ", ",\n\t\t\"%s\" : ", "\n\t}\n]\n", "\"", "null");
2532 }
2533 } else {
2534 res = mvc_export_table(b, s, t, order, 0, count, "[ ", ",\t", "\t]\n", "\"", "NULL");
2535 }
2536 BBPunfix(order->batCacheid);
2537 if (clean)
2538 m->results = res_tables_remove(m->results, t);
2539
2540 if (res > 0)
2541 res = mvc_export_warning(s, "");
2542 return res;
2543}
2544
2545
2546int
2547mvc_export_chunk(backend *b, stream *s, int res_id, BUN offset, BUN nr)
2548{
2549 mvc *m = b->mvc;
2550 int res = 0;
2551 res_table *t = res_tables_find(m->results, res_id);
2552 BAT *order = NULL;
2553 BUN cnt;
2554
2555 if (!s || !t)
2556 return 0;
2557
2558 order = BATdescriptor(t->order);
2559 if (!order)
2560 return -1;
2561 cnt = nr;
2562 if (cnt == 0)
2563 cnt = BATcount(order);
2564 if (offset >= BATcount(order))
2565 cnt = 0;
2566 if (offset + cnt > BATcount(order))
2567 cnt = BATcount(order) - offset;
2568
2569 if (b->client->protocol != PROTOCOL_10) {
2570 /* query type: Q_BLOCK */
2571 if (!(mnstr_write(s, "&6 ", 3, 1) == 1))
2572 return export_error(order);
2573
2574 /* result id */
2575 if (!mvc_send_int(s, res_id) || mnstr_write(s, " ", 1, 1) != 1)
2576 return export_error(order);
2577
2578 /* column count */
2579 if (!mvc_send_int(s, t->nr_cols) || mnstr_write(s, " ", 1, 1) != 1)
2580 return export_error(order);
2581
2582 /* row count */
2583 if (!mvc_send_lng(s, (lng) cnt) || mnstr_write(s, " ", 1, 1) != 1)
2584 return export_error(order);
2585
2586 /* block offset */
2587 if (!mvc_send_lng(s, (lng) offset))
2588 return export_error(order);
2589
2590 if (mnstr_write(s, "\n", 1, 1) != 1)
2591 return export_error(order);
2592 }
2593
2594 res = mvc_export_table(b, s, t, order, offset, cnt, "[ ", ",\t", "\t]\n", "\"", "NULL");
2595 BBPunfix(order->batCacheid);
2596 return res;
2597}
2598
2599
2600int
2601mvc_result_table(mvc *m, oid query_id, int nr_cols, sql_query_t type, BAT *order)
2602{
2603 res_table *t = res_table_create(m->session->tr, m->result_id++, query_id, nr_cols, type, m->results, order);
2604 m->results = t;
2605 if(t)
2606 return t->id;
2607 else
2608 return -1;
2609}
2610
2611int
2612mvc_result_column(mvc *m, char *tn, char *name, char *typename, int digits, int scale, BAT *b)
2613{
2614 /* return 0 on success, non-zero on failure */
2615 return res_col_create(m->session->tr, m->results, tn, name, typename, digits, scale, TYPE_bat, b) == NULL;
2616}
2617
2618int
2619mvc_result_value(mvc *m, const char *tn, const char *name, const char *typename, int digits, int scale, ptr *p, int mtype)
2620{
2621 /* return 0 on success, non-zero on failure */
2622 return res_col_create(m->session->tr, m->results, tn, name, typename, digits, scale, mtype, p) == NULL;
2623}
2624