1/*
2 * particle_geojson.c
3 *
4 * Copyright (C) 2015 Aerospike, Inc.
5 *
6 * Portions may be licensed to Aerospike, Inc. under one or more contributor
7 * license agreements.
8 *
9 * This program is free software: you can redistribute it and/or modify it under
10 * the terms of the GNU Affero General Public License as published by the Free
11 * Software Foundation, either version 3 of the License, or (at your option) any
12 * later version.
13 *
14 * This program is distributed in the hope that it will be useful, but WITHOUT
15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
16 * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
17 * details.
18 *
19 * You should have received a copy of the GNU Affero General Public License
20 * along with this program. If not, see http://www.gnu.org/licenses/
21 */
22
23
24#include <stddef.h>
25#include <stdint.h>
26#include <string.h>
27
28#include "aerospike/as_geojson.h"
29#include "aerospike/as_msgpack.h"
30#include "aerospike/as_val.h"
31#include "citrusleaf/alloc.h"
32#include "citrusleaf/cf_byte_order.h"
33
34#include "fault.h"
35
36#include "base/datamodel.h"
37#include "base/particle.h"
38#include "base/particle_blob.h"
39#include "base/proto.h"
40#include "geospatial/geospatial.h"
41
42
43//==========================================================
44// GEOJSON particle interface - function declarations.
45//
46
47// Most GEOJSON particle table functions just use the equivalent BLOB particle
48// functions. Here are the differences...
49
50// Handle "wire" format.
51int32_t geojson_concat_size_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp);
52int geojson_append_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp);
53int geojson_prepend_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp);
54int geojson_incr_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp);
55int32_t geojson_size_from_wire(const uint8_t *wire_value, uint32_t value_size);
56int geojson_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp);
57uint32_t geojson_to_wire(const as_particle *p, uint8_t *wire);
58
59// Handle as_val translation.
60uint32_t geojson_size_from_asval(const as_val *val);
61void geojson_from_asval(const as_val *val, as_particle **pp);
62as_val *geojson_to_asval(const as_particle *p);
63uint32_t geojson_asval_wire_size(const as_val *val);
64uint32_t geojson_asval_to_wire(const as_val *val, uint8_t *wire);
65
66// Handle msgpack translation.
67uint32_t geojson_size_from_msgpack(const uint8_t *packed, uint32_t packed_size);
68void geojson_from_msgpack(const uint8_t *packed, uint32_t packed_size, as_particle **pp);
69
70
71//==========================================================
72// GEOJSON particle interface - vtable.
73//
74
75const as_particle_vtable geojson_vtable = {
76 blob_destruct,
77 blob_size,
78
79 geojson_concat_size_from_wire,
80 geojson_append_from_wire,
81 geojson_prepend_from_wire,
82 geojson_incr_from_wire,
83 geojson_size_from_wire,
84 geojson_from_wire,
85 blob_compare_from_wire,
86 blob_wire_size,
87 geojson_to_wire,
88
89 geojson_size_from_asval,
90 geojson_from_asval,
91 geojson_to_asval,
92 geojson_asval_wire_size,
93 geojson_asval_to_wire,
94
95 geojson_size_from_msgpack,
96 geojson_from_msgpack,
97
98 blob_skip_flat,
99 blob_cast_from_flat,
100 blob_from_flat,
101 blob_flat_size,
102 blob_to_flat
103};
104
105
106//==========================================================
107// Typedefs & constants.
108//
109
110// GEOJSON particle flag bit-fields.
111#define GEOJSON_ISREGION 0x1
112
113// The GEOJSON particle structs overlay the related BLOB structs.
114
115typedef struct geojson_mem_s {
116 uint8_t type; // IMPORTANT: overlay blob_mem!
117 uint32_t sz; // IMPORTANT: overlay blob_mem!
118 uint8_t flags;
119 uint16_t ncells;
120 uint8_t data[]; // (ncells * uint64_t) + jsonstr
121} __attribute__ ((__packed__)) geojson_mem;
122
123typedef struct geojson_flat_s {
124 uint8_t type; // IMPORTANT: overlay blob_flat!
125 uint32_t size; // IMPORTANT: overlay blob_flat!
126 uint8_t flags;
127 uint16_t ncells;
128 uint8_t data[]; // (ncells * uint64_t) + jsonstr
129} __attribute__ ((__packed__)) geojson_flat;
130
131
132//==========================================================
133// Forward declarations.
134//
135
136static bool geojson_match(bool particle_is_region, uint64_t particle_cellid, geo_region_t particle_region, uint64_t query_cellid, geo_region_t query_region, bool is_strict);
137static inline uint32_t geojson_mem_sz(uint32_t ncells, size_t jlen);
138static inline uint32_t geojson_particle_sz(uint32_t ncells, size_t jlen);
139static inline bool geojson_parse(const char *json, uint32_t jlen, uint64_t *cellid, geo_region_t *region);
140static bool geojson_to_particle(const char *json, uint32_t jlen, as_particle **pp);
141
142
143//==========================================================
144// GEOJSON particle interface - function definitions.
145//
146
147// Most GEOJSON particle table functions just use the equivalent BLOB particle
148// functions. Here are the differences...
149
150//------------------------------------------------
151// Handle "wire" format.
152//
153
154int32_t
155geojson_concat_size_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp)
156{
157 cf_warning(AS_PARTICLE, "invalid operation on geojson particle");
158 return -1;
159}
160
161int32_t
162geojson_append_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp)
163{
164 cf_warning(AS_PARTICLE, "invalid operation on geojson particle");
165 return -1;
166}
167
168int32_t
169geojson_prepend_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp)
170{
171 cf_warning(AS_PARTICLE, "invalid operation on geojson particle");
172 return -1;
173}
174
175int32_t
176geojson_incr_from_wire(as_particle_type wire_type, const uint8_t *wire_value, uint32_t value_size, as_particle **pp)
177{
178 cf_warning(AS_PARTICLE, "invalid operation on geojson particle");
179 return -1;
180}
181
182int32_t
183geojson_size_from_wire(const uint8_t *wire_value, uint32_t value_size)
184{
185 // NOTE - Unfortunately we would need to run the JSON parser and region
186 // coverer to find out exactly how many cells we need to allocate for this
187 // particle.
188 //
189 // For now we always allocate the maximum number of cells (MAX_REGION_CELLS)
190 // for the in-memory particle.
191 //
192 // For now also ignore any incoming cells entirely.
193
194 const uint16_t *p_cells = (const uint16_t *)(wire_value + 1);
195 uint16_t ncells = cf_swap_from_be16(*p_cells);
196 size_t cellsz = ncells * sizeof(uint64_t);
197
198 if ((size_t)value_size < sizeof(uint8_t) + sizeof(uint16_t) + cellsz) {
199 cf_warning(AS_PARTICLE, "geojson_size_from_wire() invalid geojson wire_sz %u < cellsz %zu + 3", value_size, cellsz);
200 return -AS_ERR_GEO_INVALID_GEOJSON;
201 }
202
203 size_t jlen = value_size - sizeof(uint8_t) - sizeof(uint16_t) - cellsz;
204
205 return (int32_t)geojson_particle_sz(MAX_REGION_CELLS, jlen);
206}
207
208int
209geojson_from_wire(as_particle_type wire_type, const uint8_t *wire_value,
210 uint32_t value_size, as_particle **pp)
211{
212 const uint16_t *p_cells = (const uint16_t *)(wire_value + 1);
213 uint16_t ncells = cf_swap_from_be16(*p_cells);
214 size_t cellsz = ncells * sizeof(uint64_t);
215 char const *json = (char const *)p_cells + sizeof(uint16_t) + cellsz;
216 size_t jlen = value_size - sizeof(uint8_t) - sizeof(uint16_t) - cellsz;
217 geojson_mem *p_geojson_mem = (geojson_mem *)*pp;
218
219 p_geojson_mem->type = wire_type;
220
221 // We ignore any incoming cells entirely.
222
223 if (! geojson_to_particle(json, jlen, pp)) {
224 cf_warning(AS_PARTICLE, "geojson_from_wire() failed");
225 return -AS_ERR_GEO_INVALID_GEOJSON;
226 }
227
228 return AS_OK;
229}
230
231uint32_t
232geojson_to_wire(const as_particle *p, uint8_t *wire)
233{
234 // Use blob routine first.
235 uint32_t sz = blob_to_wire(p, wire);
236
237 // Swap ncells.
238 uint16_t *p_ncells = (uint16_t *)(wire + sizeof(uint8_t));
239 uint16_t ncells = *p_ncells;
240
241 *p_ncells = cf_swap_to_be16(*p_ncells);
242 ++p_ncells;
243
244 // Swap the cells.
245 uint64_t *p_cell_begin = (uint64_t *)p_ncells;
246 uint64_t *p_cell_end = p_cell_begin + ncells;
247
248 for (uint64_t *p_cell = p_cell_begin; p_cell < p_cell_end; ++p_cell) {
249 *p_cell = cf_swap_to_be64(*p_cell);
250 }
251
252 return sz;
253}
254
255//------------------------------------------------
256// Handle as_val translation.
257//
258
259uint32_t
260geojson_size_from_asval(const as_val *val)
261{
262 as_geojson *pg = as_geojson_fromval(val);
263 size_t jsz = as_geojson_len(pg);
264
265 return geojson_particle_sz(MAX_REGION_CELLS, jsz);
266}
267
268void
269geojson_from_asval(const as_val *val, as_particle **pp)
270{
271 geojson_mem *p_geojson_mem = (geojson_mem *)*pp;
272 as_geojson *pg = as_geojson_fromval(val);
273 size_t jlen = as_geojson_len(pg);
274
275 p_geojson_mem->type = AS_PARTICLE_TYPE_GEOJSON;
276
277 if (! geojson_to_particle(as_geojson_get(pg), jlen, pp)) {
278 cf_warning(AS_PARTICLE, "geojson_from_asval() failed");
279 }
280}
281
282as_val *
283geojson_to_asval(const as_particle *p)
284{
285 size_t jlen;
286 char const *json = as_geojson_mem_jsonstr(p, &jlen);
287 char *buf = cf_malloc(jlen + 1);
288
289 memcpy(buf, json, jlen);
290 buf[jlen] = '\0';
291
292 return (as_val *)as_geojson_new_wlen(buf, jlen, true);
293}
294
295uint32_t
296geojson_asval_wire_size(const as_val *val)
297{
298 as_geojson *pg = as_geojson_fromval(val);
299 size_t jlen = as_geojson_len(pg);
300
301 // We won't be writing any cellids ...
302 return geojson_mem_sz(0, jlen);
303}
304
305uint32_t
306geojson_asval_to_wire(const as_val *val, uint8_t *wire)
307{
308 as_geojson *pg = as_geojson_fromval(val);
309 size_t jlen = as_geojson_len(pg);
310
311 uint8_t *p8 = wire;
312
313 *p8++ = 0; // flags
314
315 uint16_t *p16 = (uint16_t *)p8;
316
317 *p16++ = cf_swap_to_be16(0); // no cells on output to client
318 p8 = (uint8_t *)p16;
319 memcpy(p8, as_geojson_get(pg), jlen);
320
321 return geojson_mem_sz(0, jlen);
322}
323
324//------------------------------------------------
325// Handle msgpack translation.
326//
327
328uint32_t
329geojson_size_from_msgpack(const uint8_t *packed, uint32_t packed_size)
330{
331 // Oversize by a few bytes doing the easy thing.
332 size_t jsz = (size_t)packed_size;
333
334 // Compute the size; we won't be writing any cellids ...
335 return geojson_particle_sz(0, jsz);
336}
337
338void
339geojson_from_msgpack(const uint8_t *packed, uint32_t packed_size, as_particle **pp)
340{
341 geojson_mem *p_geojson_mem = (geojson_mem *)*pp;
342
343 as_unpacker pk = {
344 .buffer = packed,
345 .offset = 0,
346 .length = packed_size
347 };
348
349 int64_t blob_size = as_unpack_blob_size(&pk);
350 const uint8_t *ptr = pk.buffer + pk.offset;
351
352 // *ptr should be AS_BYTES_GEOJSON at this point.
353
354 // Adjust for type (1 byte).
355 ptr++;
356 blob_size--;
357
358 size_t jsz = (size_t)blob_size;
359
360 p_geojson_mem->type = AS_PARTICLE_TYPE_GEOJSON;
361 p_geojson_mem->sz = geojson_mem_sz(0, jsz);
362 p_geojson_mem->flags = 0;
363 p_geojson_mem->ncells = 0;
364
365 uint8_t *p8 = (uint8_t *)p_geojson_mem->data;
366 memcpy(p8, ptr, jsz);
367}
368
369
370//==========================================================
371// Particle functions specific to GEOJSON.
372//
373
374size_t
375as_bin_particle_geojson_cellids(const as_bin *b, uint64_t **ppcells)
376{
377 geojson_mem *gp = (geojson_mem *)b->particle;
378
379 *ppcells = (uint64_t *)gp->data;
380
381 return (size_t)gp->ncells;
382}
383
384bool
385as_particle_geojson_match(as_particle *particle, uint64_t query_cellid,
386 geo_region_t query_region, bool is_strict)
387{
388 // Determine whether the candidate particle geometry is a match
389 // for the query geometry.
390 //
391 // If query_cellid is non-zero this is a regions-containing-point query.
392 //
393 // If query_region is non-null this is a points-in-region query.
394 //
395 // Candidate geometry can either be a point or a region. Regions
396 // will have the GEOJSON_ISREGION flag set.
397
398 geojson_mem *p_geojson_mem = (geojson_mem *)particle;
399 uint64_t *cells = (uint64_t *)p_geojson_mem->data;
400 uint64_t candidate_cellid = p_geojson_mem->ncells == 0 ? 0 : cells[0];
401 geo_region_t candidate_region = NULL;
402 bool candidate_is_region = (p_geojson_mem->flags & GEOJSON_ISREGION) != 0;
403
404 // If we are a strict RCP query on a region candidate we need to
405 // run the parser to obtain a candidate_region for the matcher.
406 if (query_cellid != 0 && candidate_is_region && is_strict) {
407 size_t jsonsz;
408 char const *jsonptr = as_geojson_mem_jsonstr(particle, &jsonsz);
409
410 if (! geo_parse(NULL, jsonptr, jsonsz, &candidate_cellid,
411 &candidate_region)) {
412 cf_warning(AS_PARTICLE, "geo_parse() failed - unexpected");
413 geo_region_destroy(candidate_region);
414 return false;
415 }
416 }
417
418 bool ismatch = geojson_match(candidate_is_region, candidate_cellid,
419 candidate_region, query_cellid, query_region, is_strict);
420
421 geo_region_destroy(candidate_region);
422
423 return ismatch;
424}
425
426bool
427as_particle_geojson_match_asval(const as_val *val, uint64_t query_cellid,
428 geo_region_t query_region, bool is_strict)
429{
430 as_geojson *pg = as_geojson_fromval(val);
431 size_t jlen = as_geojson_len(pg);
432 const char *json = as_geojson_get(pg);
433
434 uint64_t candidate_cellid = 0;
435 geo_region_t candidate_region = NULL;
436
437 if (! geo_parse(NULL, json, jlen, &candidate_cellid, &candidate_region)) {
438 cf_warning(AS_PARTICLE, "geo_parse() failed - unexpected");
439 geo_region_destroy(candidate_region);
440 return false;
441 }
442
443 bool ismatch = geojson_match(candidate_cellid == 0, candidate_cellid,
444 candidate_region, query_cellid, query_region, is_strict);
445
446 geo_region_destroy(candidate_region);
447
448 return ismatch;
449}
450
451const char *
452as_geojson_mem_jsonstr(const as_particle *particle, size_t *p_jlen)
453{
454 const geojson_mem *p_geojson_mem = (const geojson_mem *)particle;
455 size_t cellsz = p_geojson_mem->ncells * sizeof(uint64_t);
456
457 *p_jlen = p_geojson_mem->sz - sizeof(uint8_t) - sizeof(uint16_t) - cellsz;
458
459 return (const char *)p_geojson_mem->data + cellsz;
460}
461
462
463//==========================================================
464// Local helpers.
465//
466
467static bool
468geojson_match(bool candidate_is_region, uint64_t candidate_cellid, geo_region_t candidate_region, uint64_t query_cellid, geo_region_t query_region, bool is_strict)
469{
470 // Determine whether the candidate geometry is a match for the
471 // query geometry.
472 //
473 // If query_cellid is non-zero this is a regions-containing-point query.
474 //
475 // If query_region is non-null this is a points-in-region query.
476 //
477 // Candidate geometry can either be a point or a region. Regions
478 // will have the GEOJSON_ISREGION flag set.
479
480 // Is this a REGIONS-CONTAINING-POINT query?
481 //
482 if (query_cellid != 0) {
483
484 if (candidate_is_region) {
485 // Candidate is a REGION.
486
487 // Shortcut, if we aren't strict just return true.
488 if (! is_strict) {
489 return true;
490 }
491
492 return geo_point_within(query_cellid, candidate_region);
493 }
494 else {
495 // Candidate is a POINT, skip it.
496 return false;
497 }
498 }
499
500 // Is this a POINTS-IN-REGION query?
501 //
502 if (query_region) {
503
504 if (candidate_is_region) {
505 // Candidate is a REGION, skip it.
506 return false;
507 }
508 else {
509 // Sanity check, make sure this geometry has been processed.
510 if (candidate_cellid == 0) {
511 cf_warning(AS_PARTICLE, "candidate cellid has no value");
512 return false;
513 }
514
515 // Candidate is a POINT.
516 if (is_strict) {
517 return geo_point_within(candidate_cellid, query_region);
518 }
519 else {
520 return true;
521 }
522 }
523 }
524
525 return false;
526}
527
528static inline uint32_t
529geojson_mem_sz(uint32_t ncells, size_t jlen)
530{
531 return (uint32_t)(
532 sizeof(uint8_t) + // flags
533 sizeof(uint16_t) + // ncells (always 0 here)
534 (ncells * sizeof(uint64_t)) + // cell array
535 jlen); // json string
536}
537
538static inline uint32_t
539geojson_particle_sz(uint32_t ncells, size_t jlen)
540{
541 return (uint32_t)(
542 sizeof(geojson_mem) +
543 (ncells * sizeof(uint64_t)) + // cell array
544 jlen); // json string
545}
546
547static inline bool
548geojson_parse(const char *json, uint32_t jlen, uint64_t *cellid,
549 geo_region_t *region)
550{
551 *cellid = 0;
552 *region = NULL;
553
554 if (! geo_parse(NULL, json, jlen, cellid, region)) {
555 cf_warning(AS_PARTICLE, "geo_parse failed");
556 return false;
557 }
558
559 if (*cellid != 0 && *region != NULL) {
560 geo_region_destroy(region);
561 cf_warning(AS_PARTICLE, "geo_parse found both point and region");
562 *cellid = 0;
563 *region = NULL;
564 return false;
565 }
566
567 if (*cellid == 0 && *region == NULL) {
568 cf_warning(AS_PARTICLE, "geo_parse found neither point nor region");
569 return false;
570 }
571
572 return true;
573}
574
575static bool
576geojson_to_particle(const char *json, uint32_t jlen, as_particle **pp)
577{
578 geojson_mem *p_geojson_mem = (geojson_mem *)*pp;
579 uint64_t cellid;
580 geo_region_t region;
581 bool ret = true;
582 uint64_t *p_outcells = (uint64_t *)p_geojson_mem->data;
583
584 if (! geojson_parse(json, jlen, &cellid, &region)) {
585 ret = false;
586 }
587
588 p_geojson_mem->flags = 0;
589
590 if (cellid) { // POINT
591 p_geojson_mem->ncells = 1;
592 p_outcells[0] = cellid;
593 }
594 else if (region) { // REGION
595 p_geojson_mem->flags |= GEOJSON_ISREGION;
596
597 int numcells;
598
599 if (! geo_region_cover(NULL, region, MAX_REGION_CELLS, p_outcells, NULL,
600 NULL, &numcells)) {
601 cf_warning(AS_PARTICLE, "geo_region_cover failed");
602 ret = false;
603 numcells = 0;
604 }
605
606 p_geojson_mem->ncells = (uint16_t)numcells;
607 geo_region_destroy(region);
608 }
609 else {
610 p_geojson_mem->ncells = 0;
611 }
612
613 uint8_t *p_out = (uint8_t *)(p_outcells + p_geojson_mem->ncells);
614
615 memcpy(p_out, json, jlen);
616
617 // Set the actual size; we will waste some space at the end of the allocated
618 // particle.
619 p_geojson_mem->sz = geojson_mem_sz(p_geojson_mem->ncells, jlen);
620
621 return ret;
622}
623