| 1 | /* |
|---|---|
| 2 | * This Source Code Form is subject to the terms of the Mozilla Public |
| 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this |
| 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| 5 | * |
| 6 | * Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V. |
| 7 | */ |
| 8 | |
| 9 | #include "monetdb_config.h" |
| 10 | #include "gdk.h" |
| 11 | #include "gdk_private.h" |
| 12 | #include "gdk_calc_private.h" |
| 13 | |
| 14 | /* |
| 15 | * All join variants produce some sort of join on two input BATs, |
| 16 | * optionally subject to up to two candidate lists. Only values in |
| 17 | * the input BATs that are mentioned in the associated candidate list |
| 18 | * (if provided) are eligible. They all return two output BATs in the |
| 19 | * first two arguments. The join operations differ in the way in |
| 20 | * which tuples from the two inputs are matched. |
| 21 | * |
| 22 | * The outputs consist of two aligned BATs (i.e. same length and same |
| 23 | * hseqbase (0@0)) that contain the OIDs of the input BATs that match. |
| 24 | * The candidate lists, if given, contain the OIDs of the associated |
| 25 | * input BAT which must be considered for matching. The input BATs |
| 26 | * must have the same type. |
| 27 | * |
| 28 | * All functions also have a parameter nil_matches which indicates |
| 29 | * whether NIL must be considered an ordinary value that can match, or |
| 30 | * whether NIL must be considered to never match. |
| 31 | * |
| 32 | * The join functions that are provided here are: |
| 33 | * BATjoin |
| 34 | * normal equi-join |
| 35 | * BATleftjoin |
| 36 | * normal equi-join, but the left output is sorted |
| 37 | * BATouterjoin |
| 38 | * equi-join, but the left output is sorted, and if there is no |
| 39 | * match for a value in the left input, there is still an output |
| 40 | * with NIL in the right output |
| 41 | * BATsemijoin |
| 42 | * equi-join, but the left output is sorted, and if there are |
| 43 | * multiple matches, only one is returned (i.e., the left output |
| 44 | * is also key) |
| 45 | * BATthetajoin |
| 46 | * theta-join: an extra operator must be provided encoded as an |
| 47 | * integer (macros JOIN_EQ, JOIN_NE, JOIN_LT, JOIN_LE, JOIN_GT, |
| 48 | * JOIN_GE); values match if the left input has the given |
| 49 | * relationship with the right input; order of the outputs is not |
| 50 | * guaranteed |
| 51 | * BATbandjoin |
| 52 | * band-join: two extra input values (c1, c2) must be provided as |
| 53 | * well as Booleans (li, hi) that indicate whether the value |
| 54 | * ranges are inclusive or not; values in the left and right |
| 55 | * inputs match if right - c1 <[=] left <[=] right + c2; if c1 or |
| 56 | * c2 is NIL, there are no matches |
| 57 | * BATrangejoin |
| 58 | * range-join: the right input consists of two aligned BATs, |
| 59 | * values match if the left value is between two corresponding |
| 60 | * right values; two extra Boolean parameters, li and hi, |
| 61 | * indicate whether equal values match |
| 62 | * |
| 63 | * In addition to these functions, there are two more functions that |
| 64 | * are closely related: |
| 65 | * BATintersect |
| 66 | * intersection: return a candidate list with OIDs of tuples in |
| 67 | * the left input whose value occurs in the right input |
| 68 | * BATdiff |
| 69 | * difference: return a candidate list with OIDs of tuples in the |
| 70 | * left input whose value does not occur in the right input |
| 71 | */ |
| 72 | |
| 73 | /* Perform a bunch of sanity checks on the inputs to a join. */ |
| 74 | static gdk_return |
| 75 | joinparamcheck(BAT *l, BAT *r1, BAT *r2, BAT *sl, BAT *sr, const char *func) |
| 76 | { |
| 77 | if (ATOMtype(l->ttype) != ATOMtype(r1->ttype) || |
| 78 | (r2 && ATOMtype(l->ttype) != ATOMtype(r2->ttype))) { |
| 79 | GDKerror("%s: inputs not compatible.\n", func); |
| 80 | return GDK_FAIL; |
| 81 | } |
| 82 | if (r2 && |
| 83 | (BATcount(r1) != BATcount(r2) || r1->hseqbase != r2->hseqbase)) { |
| 84 | GDKerror("%s: right inputs not aligned.\n", func); |
| 85 | return GDK_FAIL; |
| 86 | } |
| 87 | if ((sl && ATOMtype(sl->ttype) != TYPE_oid) || |
| 88 | (sr && ATOMtype(sr->ttype) != TYPE_oid)) { |
| 89 | GDKerror("%s: candidate lists must have type OID.\n", func); |
| 90 | return GDK_FAIL; |
| 91 | } |
| 92 | if ((sl && !BATtordered(sl)) || |
| 93 | (sr && !BATtordered(sr))) { |
| 94 | GDKerror("%s: candidate lists must be sorted.\n", func); |
| 95 | return GDK_FAIL; |
| 96 | } |
| 97 | if ((sl && !BATtkey(sl)) || |
| 98 | (sr && !BATtkey(sr))) { |
| 99 | GDKerror("%s: candidate lists must be unique.\n", func); |
| 100 | return GDK_FAIL; |
| 101 | } |
| 102 | return GDK_SUCCEED; |
| 103 | } |
| 104 | |
| 105 | /* Create the result bats for a join, returns the absolute maximum |
| 106 | * number of outputs that could possibly be generated. */ |
| 107 | static BUN |
| 108 | joininitresults(BAT **r1p, BAT **r2p, BUN lcnt, BUN rcnt, bool lkey, bool rkey, |
| 109 | bool semi, bool nil_on_miss, bool only_misses, BUN estimate) |
| 110 | { |
| 111 | BAT *r1, *r2; |
| 112 | BUN maxsize, size; |
| 113 | |
| 114 | /* if nil_on_miss is set, we really need a right output */ |
| 115 | assert(!nil_on_miss || r2p != NULL); |
| 116 | |
| 117 | lkey |= lcnt <= 1; |
| 118 | rkey |= rcnt <= 1; |
| 119 | |
| 120 | *r1p = NULL; |
| 121 | if (r2p) |
| 122 | *r2p = NULL; |
| 123 | if (lcnt == 0) { |
| 124 | /* there is nothing to match */ |
| 125 | maxsize = 0; |
| 126 | } else if (!only_misses && !nil_on_miss && rcnt == 0) { |
| 127 | /* if right is empty, we have no hits, so if we don't |
| 128 | * want misses, the result is empty */ |
| 129 | maxsize = 0; |
| 130 | } else if (rkey | semi | only_misses) { |
| 131 | /* each entry left matches at most one on right, in |
| 132 | * case nil_on_miss is also set, each entry matches |
| 133 | * exactly one (see below) */ |
| 134 | maxsize = lcnt; |
| 135 | } else if (lkey) { |
| 136 | /* each entry on right is matched at most once */ |
| 137 | if (nil_on_miss) { |
| 138 | /* one entry left could match all right, and |
| 139 | * all other entries left match nil */ |
| 140 | maxsize = lcnt + rcnt - 1; |
| 141 | } else { |
| 142 | maxsize = rcnt; |
| 143 | } |
| 144 | } else if (rcnt == 0) { |
| 145 | /* nil_on_miss must be true due to previous checks, so |
| 146 | * all values on left miss */ |
| 147 | maxsize = lcnt; |
| 148 | } else if (BUN_MAX / lcnt >= rcnt) { |
| 149 | /* in the worst case we have a full cross product */ |
| 150 | maxsize = lcnt * rcnt; |
| 151 | } else { |
| 152 | /* a BAT cannot grow larger than BUN_MAX */ |
| 153 | maxsize = BUN_MAX; |
| 154 | } |
| 155 | size = estimate == BUN_NONE ? lcnt < rcnt ? lcnt : rcnt : estimate; |
| 156 | if (size < 1024) |
| 157 | size = 1024; |
| 158 | if (size > maxsize) |
| 159 | size = maxsize; |
| 160 | if ((rkey | semi | only_misses) & nil_on_miss) { |
| 161 | /* see comment above: each entry left matches exactly |
| 162 | * once */ |
| 163 | size = maxsize; |
| 164 | } |
| 165 | |
| 166 | if (maxsize == 0) { |
| 167 | r1 = BATdense(0, 0, 0); |
| 168 | if (r1 == NULL) { |
| 169 | return BUN_NONE; |
| 170 | } |
| 171 | if (r2p) { |
| 172 | r2 = BATdense(0, 0, 0); |
| 173 | if (r2 == NULL) { |
| 174 | BBPreclaim(r1); |
| 175 | return BUN_NONE; |
| 176 | } |
| 177 | *r2p = r2; |
| 178 | } |
| 179 | *r1p = r1; |
| 180 | return 0; |
| 181 | } |
| 182 | |
| 183 | r1 = COLnew(0, TYPE_oid, size, TRANSIENT); |
| 184 | if (r1 == NULL) { |
| 185 | return BUN_NONE; |
| 186 | } |
| 187 | r1->tnil = false; |
| 188 | r1->tnonil = true; |
| 189 | r1->tkey = true; |
| 190 | r1->tsorted = true; |
| 191 | r1->trevsorted = true; |
| 192 | r1->tseqbase = 0; |
| 193 | *r1p = r1; |
| 194 | if (r2p) { |
| 195 | r2 = COLnew(0, TYPE_oid, size, TRANSIENT); |
| 196 | if (r2 == NULL) { |
| 197 | BBPreclaim(r1); |
| 198 | return BUN_NONE; |
| 199 | } |
| 200 | r2->tnil = false; |
| 201 | r2->tnonil = true; |
| 202 | r2->tkey = true; |
| 203 | r2->tsorted = true; |
| 204 | r2->trevsorted = true; |
| 205 | r2->tseqbase = 0; |
| 206 | *r2p = r2; |
| 207 | } |
| 208 | return maxsize; |
| 209 | } |
| 210 | |
| 211 | #define VALUE(s, x) (s##vars ? \ |
| 212 | s##vars + VarHeapVal(s##vals, (x), s##width) : \ |
| 213 | s##vals ? (const char *) s##vals + ((x) * s##width) : \ |
| 214 | (s##val = BUNtoid(s, (x)), (const char *) &s##val)) |
| 215 | #define FVALUE(s, x) ((const char *) s##vals + ((x) * s##width)) |
| 216 | |
| 217 | #define APPEND(b, o) (((oid *) b->theap.base)[b->batCount++] = (o)) |
| 218 | |
| 219 | #define MAYBEEXTEND_PROGRESS(CNT, LCUR, LCNT) \ |
| 220 | do { \ |
| 221 | BUN N = (CNT); \ |
| 222 | if (BATcount(r1) + N > BATcapacity(r1)) { \ |
| 223 | /* make some extra space by extrapolating how */ \ |
| 224 | /* much more we need (fraction of l we've seen */ \ |
| 225 | /* so far is used as the fraction of the */ \ |
| 226 | /* expected result size we've produced so */ \ |
| 227 | /* far) */ \ |
| 228 | BUN newcap = (BUN) ((double) (LCNT) / (LCUR) * (BATcount(r1) + N) * 1.5); \ |
| 229 | if (newcap < N + BATcount(r1)) \ |
| 230 | newcap = N + BATcount(r1) + 1024; \ |
| 231 | if (newcap > maxsize) \ |
| 232 | newcap = maxsize; \ |
| 233 | /* make sure heap.free is set properly before \ |
| 234 | * extending */ \ |
| 235 | BATsetcount(r1, BATcount(r1)); \ |
| 236 | if (BATextend(r1, newcap) != GDK_SUCCEED) \ |
| 237 | goto bailout; \ |
| 238 | if (r2) { \ |
| 239 | BATsetcount(r2, BATcount(r2)); \ |
| 240 | if (BATextend(r2, newcap) != GDK_SUCCEED) \ |
| 241 | goto bailout; \ |
| 242 | assert(BATcapacity(r1) == BATcapacity(r2)); \ |
| 243 | } \ |
| 244 | } \ |
| 245 | } while (0) |
| 246 | |
| 247 | #define MAYBEEXTEND(CNT, CI) MAYBEEXTEND_PROGRESS(CNT, (CI)->next, (CI)->ncand) |
| 248 | #define MAYBEEXTEND_NO_CAND(CNT) MAYBEEXTEND_PROGRESS(CNT, lstart, lend) |
| 249 | |
| 250 | /* Return BATs through r1p and r2p for the case that there is no |
| 251 | * match between l and r, taking all flags into consideration. |
| 252 | * |
| 253 | * This means, if nil_on_miss is set or only_misses is set, *r1p is a |
| 254 | * copy of the left candidate list or a dense list of all "head" |
| 255 | * values of l, and *r2p (if r2p is not NULL) is all nil. If neither |
| 256 | * of those flags is set, the result is two empty BATs. */ |
| 257 | static gdk_return |
| 258 | nomatch(BAT **r1p, BAT **r2p, BAT *l, BAT *r, struct canditer *restrict lci, |
| 259 | bool nil_on_miss, bool only_misses, const char *func, lng t0) |
| 260 | { |
| 261 | BAT *r1, *r2 = NULL; |
| 262 | |
| 263 | if (lci->ncand == 0 || !(nil_on_miss | only_misses)) { |
| 264 | /* return empty BATs */ |
| 265 | if ((r1 = BATdense(0, 0, 0)) == NULL) |
| 266 | return GDK_FAIL; |
| 267 | if (r2p) { |
| 268 | if ((r2 = BATdense(0, 0, 0)) == NULL) { |
| 269 | BBPreclaim(r1); |
| 270 | return GDK_FAIL; |
| 271 | } |
| 272 | *r2p = r2; |
| 273 | } |
| 274 | *r1p = r1; |
| 275 | ALGODEBUG fprintf(stderr, |
| 276 | "#%s(l=%s,r=%s)=("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us -- nomatch\n", |
| 277 | func, BATgetId(l), BATgetId(r), |
| 278 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 279 | GDKusec() - t0); |
| 280 | return GDK_SUCCEED; |
| 281 | } |
| 282 | |
| 283 | r1 = canditer_slice(lci, 0, lci->ncand); |
| 284 | if (r2p) { |
| 285 | if ((r2 = BATconstant(0, TYPE_void, &oid_nil, lci->ncand, TRANSIENT)) == NULL) { |
| 286 | BBPreclaim(r1); |
| 287 | return GDK_FAIL; |
| 288 | } |
| 289 | *r2p = r2; |
| 290 | } |
| 291 | *r1p = r1; |
| 292 | ALGODEBUG fprintf(stderr, |
| 293 | "#%s(l=%s,r=%s)=("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us -- nomatch\n", |
| 294 | func, BATgetId(l), BATgetId(r), |
| 295 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 296 | GDKusec() - t0); |
| 297 | return GDK_SUCCEED; |
| 298 | } |
| 299 | |
| 300 | /* Implementation of join where there is a single value (possibly |
| 301 | * repeated multiple times) on the left. This means we can use a |
| 302 | * point select to find matches in the right column. */ |
| 303 | static gdk_return |
| 304 | selectjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, |
| 305 | struct canditer *restrict lci, bool nil_matches, lng t0, |
| 306 | bool swapped, const char *reason) |
| 307 | { |
| 308 | BATiter li = bat_iterator(l); |
| 309 | const void *v; |
| 310 | BAT *bn = NULL; |
| 311 | |
| 312 | assert(lci->ncand > 0); |
| 313 | assert(lci->ncand == 1 || (l->tsorted && l->trevsorted)); |
| 314 | |
| 315 | oid o = canditer_next(lci); |
| 316 | v = BUNtail(li, o - l->hseqbase); |
| 317 | |
| 318 | if (!nil_matches && |
| 319 | (*ATOMcompare(l->ttype))(v, ATOMnilptr(l->ttype)) == 0) { |
| 320 | /* NIL doesn't match anything */ |
| 321 | return nomatch(r1p, r2p, l, r, lci, false, false, |
| 322 | "selectjoin", t0); |
| 323 | } |
| 324 | |
| 325 | bn = BATselect(r, sr, v, NULL, true, true, false); |
| 326 | if (bn == NULL) { |
| 327 | return GDK_FAIL; |
| 328 | } |
| 329 | if (BATcount(bn) == 0) { |
| 330 | BBPunfix(bn->batCacheid); |
| 331 | return nomatch(r1p, r2p, l, r, lci, false, false, |
| 332 | "selectjoin", t0); |
| 333 | } |
| 334 | BAT *r1 = COLnew(0, TYPE_oid, lci->ncand * BATcount(bn), TRANSIENT); |
| 335 | if (r1 == NULL) { |
| 336 | BBPunfix(bn->batCacheid); |
| 337 | return GDK_FAIL; |
| 338 | } |
| 339 | BAT *r2 = NULL; |
| 340 | if (r2p) { |
| 341 | r2 = COLnew(0, TYPE_oid, lci->ncand * BATcount(bn), TRANSIENT); |
| 342 | if (r2 == NULL) { |
| 343 | BBPunfix(bn->batCacheid); |
| 344 | BBPreclaim(r1); |
| 345 | return GDK_FAIL; |
| 346 | } |
| 347 | } |
| 348 | |
| 349 | r1->tsorted = true; |
| 350 | r1->trevsorted = lci->ncand == 1; |
| 351 | r1->tseqbase = BATcount(bn) == 1 && lci->tpe == cand_dense ? o : oid_nil; |
| 352 | r1->tkey = BATcount(bn) == 1; |
| 353 | r1->tnil = false; |
| 354 | r1->tnonil = true; |
| 355 | if (r2) { |
| 356 | r2->tsorted = lci->ncand == 1 || BATcount(bn) == 1; |
| 357 | r2->trevsorted = BATcount(bn) == 1; |
| 358 | r2->tseqbase = lci->ncand == 1 && BATtdense(bn) ? bn->tseqbase : oid_nil; |
| 359 | r2->tkey = lci->ncand == 1; |
| 360 | r2->tnil = false; |
| 361 | r2->tnonil = true; |
| 362 | } |
| 363 | if (BATtdense(bn)) { |
| 364 | oid *o1p = (oid *) Tloc(r1, 0); |
| 365 | oid *o2p = r2 ? (oid *) Tloc(r2, 0) : NULL; |
| 366 | oid bno = bn->tseqbase; |
| 367 | BUN p, q = BATcount(bn); |
| 368 | |
| 369 | do { |
| 370 | for (p = 0; p < q; p++) { |
| 371 | *o1p++ = o; |
| 372 | } |
| 373 | if (o2p) { |
| 374 | for (p = 0; p < q; p++) { |
| 375 | *o2p++ = bno + p; |
| 376 | } |
| 377 | } |
| 378 | o = canditer_next(lci); |
| 379 | } while (!is_oid_nil(o)); |
| 380 | } else { |
| 381 | oid *o1p = (oid *) Tloc(r1, 0); |
| 382 | oid *o2p = r2 ? (oid *) Tloc(r2, 0) : NULL; |
| 383 | const oid *bnp = (const oid *) Tloc(bn, 0); |
| 384 | BUN p, q = BATcount(bn); |
| 385 | |
| 386 | do { |
| 387 | for (p = 0; p < q; p++) { |
| 388 | *o1p++ = o; |
| 389 | } |
| 390 | if (o2p) { |
| 391 | for (p = 0; p < q; p++) { |
| 392 | *o2p++ = bnp[p]; |
| 393 | } |
| 394 | } |
| 395 | o = canditer_next(lci); |
| 396 | } while (!is_oid_nil(o)); |
| 397 | } |
| 398 | BATsetcount(r1, lci->ncand * BATcount(bn)); |
| 399 | *r1p = r1; |
| 400 | if (r2p) { |
| 401 | BATsetcount(r2, lci->ncand * BATcount(bn)); |
| 402 | *r2p = r2; |
| 403 | } |
| 404 | BBPunfix(bn->batCacheid); |
| 405 | ALGODEBUG fprintf(stderr, "#%s: %s(l="ALGOBATFMT "," |
| 406 | "r="ALGOBATFMT ",sl="ALGOOPTBATFMT "," |
| 407 | "sr="ALGOOPTBATFMT ",nil_matches=%d)%s %s " |
| 408 | "-> ("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 409 | MT_thread_getname(), __func__, |
| 410 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 411 | ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr), |
| 412 | nil_matches, |
| 413 | swapped ? " swapped": "", reason, |
| 414 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 415 | GDKusec() - t0); |
| 416 | |
| 417 | return GDK_SUCCEED; |
| 418 | } |
| 419 | |
| 420 | #if SIZEOF_OID == SIZEOF_INT |
| 421 | #define binsearch_oid(indir, offset, vals, lo, hi, v, ordering, last) binsearch_int(indir, offset, (const int *) vals, lo, hi, (int) (v), ordering, last) |
| 422 | #endif |
| 423 | #if SIZEOF_OID == SIZEOF_LNG |
| 424 | #define binsearch_oid(indir, offset, vals, lo, hi, v, ordering, last) binsearch_lng(indir, offset, (const lng *) vals, lo, hi, (lng) (v), ordering, last) |
| 425 | #endif |
| 426 | |
| 427 | /* Implementation of join where the right-hand side is dense, and if |
| 428 | * there is a right candidate list, it too is dense. In case |
| 429 | * nil_on_miss is not set, we use a range select (BATselect) to find |
| 430 | * the matching values in the left column and then calculate the |
| 431 | * corresponding matches from the right. If nil_on_miss is set, we |
| 432 | * need to do some more work. */ |
| 433 | static gdk_return |
| 434 | mergejoin_void(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, |
| 435 | struct canditer *restrict lci, struct canditer *restrict rci, |
| 436 | bool nil_on_miss, bool only_misses, lng t0, bool swapped, |
| 437 | const char *reason) |
| 438 | { |
| 439 | oid lo, hi; |
| 440 | BUN i; |
| 441 | oid o, *o1p = NULL, *o2p = NULL; |
| 442 | BAT *r1 = NULL, *r2 = NULL; |
| 443 | const oid *lvals = NULL; |
| 444 | |
| 445 | /* r is dense, and if there is a candidate list, it too is |
| 446 | * dense. This means we don't have to do any searches, we |
| 447 | * only need to compare ranges to know whether a value from l |
| 448 | * has a match in r */ |
| 449 | assert(ATOMtype(l->ttype) == ATOMtype(r->ttype)); |
| 450 | assert(r->tsorted || r->trevsorted); |
| 451 | assert(BATcount(l) > 0); |
| 452 | assert(rci->tpe == cand_dense); |
| 453 | assert(BATcount(r) > 0); |
| 454 | |
| 455 | /* figure out range [lo..hi) of values in r that we need to match */ |
| 456 | lo = r->tseqbase; |
| 457 | hi = lo + BATcount(r); |
| 458 | /* restrict [lo..hi) range further using candidate list */ |
| 459 | if (rci->seq > r->hseqbase) |
| 460 | lo += rci->seq - r->hseqbase; |
| 461 | if (rci->seq + rci->ncand < r->hseqbase + BATcount(r)) |
| 462 | hi -= r->hseqbase + BATcount(r) - rci->seq - rci->ncand; |
| 463 | |
| 464 | /* at this point, the matchable values in r are [lo..hi) */ |
| 465 | if (!nil_on_miss) { |
| 466 | r1 = BATselect(l, sl, &lo, &hi, true, false, only_misses); |
| 467 | if (r1 == NULL) |
| 468 | return GDK_FAIL; |
| 469 | if (only_misses && !l->tnonil) { |
| 470 | /* also look for NILs */ |
| 471 | r2 = BATselect(l, sl, &oid_nil, NULL, true, false, false); |
| 472 | if (r2 == NULL) { |
| 473 | BBPreclaim(r1); |
| 474 | return GDK_FAIL; |
| 475 | } |
| 476 | if (BATcount(r2) > 0) { |
| 477 | BAT *mg = BATmergecand(r1, r2); |
| 478 | BBPunfix(r1->batCacheid); |
| 479 | BBPunfix(r2->batCacheid); |
| 480 | r1 = mg; |
| 481 | if (r1 == NULL) |
| 482 | return GDK_FAIL; |
| 483 | } else { |
| 484 | BBPunfix(r2->batCacheid); |
| 485 | } |
| 486 | r2 = NULL; |
| 487 | } |
| 488 | *r1p = r1; |
| 489 | if (r2p == NULL) |
| 490 | goto doreturn2; |
| 491 | if (BATcount(r1) == 0) { |
| 492 | r2 = BATdense(0, 0, 0); |
| 493 | if (r2 == NULL) { |
| 494 | BBPreclaim(r1); |
| 495 | return GDK_FAIL; |
| 496 | } |
| 497 | } else if (BATtdense(r1) && BATtdense(l)) { |
| 498 | r2 = BATdense(0, l->tseqbase + r1->tseqbase - l->hseqbase + r->hseqbase - r->tseqbase, BATcount(r1)); |
| 499 | if (r2 == NULL) { |
| 500 | BBPreclaim(r1); |
| 501 | return GDK_FAIL; |
| 502 | } |
| 503 | } else { |
| 504 | r2 = COLnew(0, TYPE_oid, BATcount(r1), TRANSIENT); |
| 505 | if (r2 == NULL) { |
| 506 | BBPreclaim(r1); |
| 507 | return GDK_FAIL; |
| 508 | } |
| 509 | const oid *lp = (const oid *) Tloc(l, 0); |
| 510 | const oid *o1p = (const oid *) Tloc(r1, 0); |
| 511 | oid *o2p = (oid *) Tloc(r2, 0); |
| 512 | hi = BATcount(r1); |
| 513 | if (l->ttype == TYPE_void && l->tvheap != NULL) { |
| 514 | /* this is actually generic code */ |
| 515 | for (o = 0; o < hi; o++) |
| 516 | o2p[o] = BUNtoid(l, BUNtoid(r1, o) - l->hseqbase) - r->tseqbase + r->hseqbase; |
| 517 | } else if (BATtdense(r1)) { |
| 518 | lo = r1->tseqbase - l->hseqbase; |
| 519 | if (r->tseqbase == r->hseqbase) { |
| 520 | memcpy(o2p, lp + lo, hi * SIZEOF_OID); |
| 521 | } else { |
| 522 | hi += lo; |
| 523 | for (o = 0; lo < hi; o++, lo++) { |
| 524 | o2p[o] = lp[lo] - r->tseqbase + r->hseqbase; |
| 525 | } |
| 526 | } |
| 527 | } else if (BATtdense(l)) { |
| 528 | for (o = 0; o < hi; o++) { |
| 529 | o2p[o] = o1p[o] - l->hseqbase + l->tseqbase - r->tseqbase + r->hseqbase; |
| 530 | } |
| 531 | } else { |
| 532 | for (o = 0; o < hi; o++) { |
| 533 | o2p[o] = lp[o1p[o] - l->hseqbase] - r->tseqbase + r->hseqbase; |
| 534 | } |
| 535 | } |
| 536 | r2->tkey = l->tkey; |
| 537 | r2->tsorted = l->tsorted; |
| 538 | r2->trevsorted = l->trevsorted; |
| 539 | r2->tnil = false; |
| 540 | r2->tnonil = true; |
| 541 | BATsetcount(r2, BATcount(r1)); |
| 542 | } |
| 543 | *r2p = r2; |
| 544 | goto doreturn2; |
| 545 | } |
| 546 | /* nil_on_miss is set, this means we must have a second output */ |
| 547 | assert(r2p); |
| 548 | if (BATtdense(l)) { |
| 549 | /* if l is dense, we can further restrict the [lo..hi) |
| 550 | * range to values in l that match with values in r */ |
| 551 | o = lo; |
| 552 | i = lci->seq - l->hseqbase; |
| 553 | if (l->tseqbase + i > lo) |
| 554 | lo = l->tseqbase + i; |
| 555 | i = canditer_last(lci) + 1 - l->hseqbase; |
| 556 | if (l->tseqbase + i < hi) |
| 557 | hi = l->tseqbase + i; |
| 558 | if (lci->tpe == cand_dense) { |
| 559 | /* l is dense, and so is the left candidate |
| 560 | * list (if it exists); this means we don't |
| 561 | * have to actually look at any values in l: |
| 562 | * we can just do some arithmetic; it also |
| 563 | * means that r1 will be dense, and if |
| 564 | * nil_on_miss is not set, or if all values in |
| 565 | * l match, r2 will too */ |
| 566 | if (hi <= lo) { |
| 567 | return nomatch(r1p, r2p, l, r, lci, |
| 568 | nil_on_miss, only_misses, |
| 569 | "mergejoin_void", t0); |
| 570 | } |
| 571 | |
| 572 | /* at this point, the matched values in l and |
| 573 | * r (taking candidate lists into account) are |
| 574 | * [lo..hi) which we can translate back to the |
| 575 | * respective OID values that we can store in |
| 576 | * r1 and r2; note that r1 will be dense since |
| 577 | * all values in l will match something (even |
| 578 | * if nil since nil_on_miss is set) */ |
| 579 | *r1p = r1 = BATdense(0, lci->seq, lci->ncand); |
| 580 | if (r1 == NULL) |
| 581 | return GDK_FAIL; |
| 582 | if (hi - lo < lci->ncand) { |
| 583 | /* we need to fill in nils in r2 for |
| 584 | * missing values */ |
| 585 | *r2p = r2 = COLnew(0, TYPE_oid, lci->ncand, TRANSIENT); |
| 586 | if (r2 == NULL) { |
| 587 | BBPreclaim(*r1p); |
| 588 | return GDK_FAIL; |
| 589 | } |
| 590 | o2p = (oid *) Tloc(r2, 0); |
| 591 | i = l->tseqbase + lci->seq - l->hseqbase; |
| 592 | lo -= i; |
| 593 | hi -= i; |
| 594 | i += r->hseqbase - r->tseqbase; |
| 595 | for (o = 0; o < lo; o++) |
| 596 | *o2p++ = oid_nil; |
| 597 | for (o = lo; o < hi; o++) |
| 598 | *o2p++ = o + i; |
| 599 | for (o = hi; o < lci->ncand; o++) |
| 600 | *o2p++ = oid_nil; |
| 601 | r2->tnonil = false; |
| 602 | r2->tnil = true; |
| 603 | /* sorted of no nils at end */ |
| 604 | r2->tsorted = hi == lci->ncand; |
| 605 | /* reverse sorted if single non-nil at start */ |
| 606 | r2->trevsorted = lo == 0 && hi == 1; |
| 607 | r2->tseqbase = oid_nil; |
| 608 | /* (hi - lo) different OIDs in r2, |
| 609 | * plus one for nil */ |
| 610 | r2->tkey = hi - lo + 1 == lci->ncand; |
| 611 | BATsetcount(r2, lci->ncand); |
| 612 | } else { |
| 613 | /* no missing values */ |
| 614 | *r2p = r2 = BATdense(0, r->hseqbase + lo - r->tseqbase, lci->ncand); |
| 615 | if (r2 == NULL) { |
| 616 | BBPreclaim(*r1p); |
| 617 | return GDK_FAIL; |
| 618 | } |
| 619 | } |
| 620 | goto doreturn; |
| 621 | } |
| 622 | /* l is dense, but the candidate list exists and is |
| 623 | * not dense; we can, by manipulating the range |
| 624 | * [lo..hi), just look at the candidate list values */ |
| 625 | |
| 626 | /* translate lo and hi to l's OID values that now need |
| 627 | * to match */ |
| 628 | lo = lo - l->tseqbase + l->hseqbase; |
| 629 | hi = hi - l->tseqbase + l->hseqbase; |
| 630 | |
| 631 | *r1p = r1 = COLnew(0, TYPE_oid, lci->ncand, TRANSIENT); |
| 632 | *r2p = r2 = COLnew(0, TYPE_oid, lci->ncand, TRANSIENT); |
| 633 | if (r1 == NULL || r2 == NULL) { |
| 634 | BBPreclaim(r1); |
| 635 | BBPreclaim(r2); |
| 636 | return GDK_FAIL; |
| 637 | } |
| 638 | o1p = (oid *) Tloc(r1, 0); |
| 639 | o2p = (oid *) Tloc(r2, 0); |
| 640 | r2->tnil = false; |
| 641 | r2->tnonil = true; |
| 642 | r2->tkey = true; |
| 643 | r2->tsorted = true; |
| 644 | o = canditer_next(lci); |
| 645 | for (i = 0; i < lci->ncand && o < lo; i++) { |
| 646 | *o1p++ = o; |
| 647 | *o2p++ = oid_nil; |
| 648 | o = canditer_next(lci); |
| 649 | } |
| 650 | if (i > 0) { |
| 651 | r2->tnil = true; |
| 652 | r2->tnonil = false; |
| 653 | r2->tkey = i == 1; |
| 654 | } |
| 655 | for (; i < lci->ncand && o < hi; i++) { |
| 656 | *o1p++ = o; |
| 657 | *o2p++ = o - l->hseqbase + l->tseqbase - r->tseqbase + r->hseqbase; |
| 658 | o = canditer_next(lci); |
| 659 | } |
| 660 | if (i < lci->ncand) { |
| 661 | r2->tkey = !r2->tnil && lci->ncand - i == 1; |
| 662 | r2->tnil = true; |
| 663 | r2->tnonil = false; |
| 664 | r2->tsorted = false; |
| 665 | for (; i < lci->ncand; i++) { |
| 666 | *o1p++ = o; |
| 667 | *o2p++ = oid_nil; |
| 668 | o = canditer_next(lci); |
| 669 | } |
| 670 | } |
| 671 | BATsetcount(r1, lci->ncand); |
| 672 | r1->tseqbase = BATcount(r1) == 1 ? *(oid*)Tloc(r1, 0) : oid_nil; |
| 673 | r1->tsorted = true; |
| 674 | r1->trevsorted = BATcount(r1) <= 1; |
| 675 | r1->tnil = false; |
| 676 | r1->tnonil = true; |
| 677 | r1->tkey = true; |
| 678 | BATsetcount(r2, BATcount(r1)); |
| 679 | r2->tseqbase = r2->tnil || BATcount(r2) > 1 ? oid_nil : BATcount(r2) == 1 ? *(oid*)Tloc(r2, 0) : 0; |
| 680 | r2->trevsorted = BATcount(r2) <= 1; |
| 681 | goto doreturn; |
| 682 | } |
| 683 | /* l is not dense, so we need to look at the values and check |
| 684 | * whether they are in the range [lo..hi) */ |
| 685 | lvals = (const oid *) Tloc(l, 0); |
| 686 | |
| 687 | /* do indirection through the candidate list to look at the |
| 688 | * value */ |
| 689 | |
| 690 | *r1p = r1 = COLnew(0, TYPE_oid, lci->ncand, TRANSIENT); |
| 691 | *r2p = r2 = COLnew(0, TYPE_oid, lci->ncand, TRANSIENT); |
| 692 | if (r1 == NULL || r2 == NULL) { |
| 693 | BBPreclaim(r1); |
| 694 | BBPreclaim(r2); |
| 695 | return GDK_FAIL; |
| 696 | } |
| 697 | o1p = (oid *) Tloc(r1, 0); |
| 698 | o2p = (oid *) Tloc(r2, 0); |
| 699 | r2->tnil = false; |
| 700 | r2->tnonil = true; |
| 701 | if (l->ttype == TYPE_void && l->tvheap != NULL) { |
| 702 | for (i = 0; i < lci->ncand; i++) { |
| 703 | oid c = canditer_next(lci); |
| 704 | |
| 705 | o = BUNtoid(l, c - l->hseqbase); |
| 706 | *o1p++ = c; |
| 707 | if (o >= lo && o < hi) { |
| 708 | *o2p++ = o - r->tseqbase + r->hseqbase; |
| 709 | } else { |
| 710 | *o2p++ = oid_nil; |
| 711 | r2->tnil = true; |
| 712 | r2->tnonil = false; |
| 713 | } |
| 714 | } |
| 715 | } else { |
| 716 | for (i = 0; i < lci->ncand; i++) { |
| 717 | oid c = canditer_next(lci); |
| 718 | |
| 719 | o = lvals[c - l->hseqbase]; |
| 720 | *o1p++ = c; |
| 721 | if (o >= lo && o < hi) { |
| 722 | *o2p++ = o - r->tseqbase + r->hseqbase; |
| 723 | } else { |
| 724 | *o2p++ = oid_nil; |
| 725 | r2->tnil = true; |
| 726 | r2->tnonil = false; |
| 727 | } |
| 728 | } |
| 729 | } |
| 730 | r1->tsorted = true; |
| 731 | r1->trevsorted = BATcount(r1) <= 1; |
| 732 | r1->tkey = true; |
| 733 | r1->tseqbase = oid_nil; |
| 734 | r1->tnil = false; |
| 735 | r1->tnonil = true; |
| 736 | BATsetcount(r1, lci->ncand); |
| 737 | BATsetcount(r2, lci->ncand); |
| 738 | r2->tsorted = l->tsorted || BATcount(r2) <= 1; |
| 739 | r2->trevsorted = l->trevsorted || BATcount(r2) <= 1; |
| 740 | r2->tkey = l->tkey || BATcount(r2) <= 1; |
| 741 | r2->tseqbase = oid_nil; |
| 742 | |
| 743 | doreturn: |
| 744 | if (r1->tkey) |
| 745 | virtualize(r1); |
| 746 | if (r2->tkey && r2->tsorted) |
| 747 | virtualize(r2); |
| 748 | doreturn2: |
| 749 | ALGODEBUG fprintf(stderr, "#%s: %s(l="ALGOBATFMT "," |
| 750 | "r="ALGOBATFMT ",sl="ALGOOPTBATFMT "," |
| 751 | "sr="ALGOOPTBATFMT "," |
| 752 | "nil_on_miss=%d,only_misses=%d)%s %s " |
| 753 | "-> ("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 754 | MT_thread_getname(), __func__, |
| 755 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 756 | ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr), |
| 757 | nil_on_miss, only_misses, |
| 758 | swapped ? " swapped": "", reason, |
| 759 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 760 | GDKusec() - t0); |
| 761 | |
| 762 | return GDK_SUCCEED; |
| 763 | } |
| 764 | |
| 765 | /* Implementation of mergejoin (see below) for the special case that |
| 766 | * the values are of type int, and some more conditions are met. */ |
| 767 | static gdk_return |
| 768 | mergejoin_int(BAT **r1p, BAT **r2p, BAT *l, BAT *r, |
| 769 | bool nil_matches, BUN estimate, lng t0, bool swapped, |
| 770 | const char *reason) |
| 771 | { |
| 772 | BAT *r1, *r2; |
| 773 | BUN lstart, lend, lcnt; |
| 774 | BUN rstart, rend; |
| 775 | BUN lscan, rscan; /* opportunistic scan window */ |
| 776 | BUN maxsize; |
| 777 | const int *lvals, *rvals; |
| 778 | int v; |
| 779 | BUN nl, nr; |
| 780 | oid lv; |
| 781 | BUN i; |
| 782 | |
| 783 | assert(ATOMtype(l->ttype) == ATOMtype(r->ttype)); |
| 784 | assert(r->tsorted || r->trevsorted); |
| 785 | |
| 786 | lstart = rstart = 0; |
| 787 | lend = BATcount(l); |
| 788 | lcnt = lend - lstart; |
| 789 | rend = BATcount(r); |
| 790 | lvals = (const int *) Tloc(l, 0); |
| 791 | rvals = (const int *) Tloc(r, 0); |
| 792 | assert(!r->tvarsized || !r->ttype); |
| 793 | |
| 794 | /* basic properties will be adjusted if necessary later on, |
| 795 | * they were initially set by joininitresults() */ |
| 796 | |
| 797 | if (lend == 0 || rend == 0) { |
| 798 | /* there are no matches */ |
| 799 | return nomatch(r1p, r2p, l, r, |
| 800 | &(struct canditer) {.tpe = cand_dense, .ncand = lcnt,}, |
| 801 | false, false, __func__, t0); |
| 802 | } |
| 803 | |
| 804 | if ((maxsize = joininitresults(r1p, r2p, BATcount(l), BATcount(r), |
| 805 | l->tkey, r->tkey, false, false, |
| 806 | false, estimate)) == BUN_NONE) |
| 807 | return GDK_FAIL; |
| 808 | r1 = *r1p; |
| 809 | r2 = r2p ? *r2p : NULL; |
| 810 | |
| 811 | /* determine opportunistic scan window for l and r */ |
| 812 | for (nl = lend - lstart, lscan = 4; nl > 0; lscan++) |
| 813 | nl >>= 1; |
| 814 | for (nr = rend - rstart, rscan = 4; nr > 0; rscan++) |
| 815 | nr >>= 1; |
| 816 | |
| 817 | if (!nil_matches) { |
| 818 | /* skip over nils at the start of the columns */ |
| 819 | if (lscan < lend - lstart && is_int_nil(lvals[lstart + lscan])) { |
| 820 | lstart = binsearch_int(NULL, 0, lvals, lstart + lscan, |
| 821 | lend - 1, int_nil, 1, 1); |
| 822 | } else { |
| 823 | while (is_int_nil(lvals[lstart])) |
| 824 | lstart++; |
| 825 | } |
| 826 | if (rscan < rend - rstart && is_int_nil(rvals[rstart + rscan])) { |
| 827 | rstart = binsearch_int(NULL, 0, rvals, rstart + rscan, |
| 828 | rend - 1, int_nil, 1, 1); |
| 829 | } else { |
| 830 | while (is_int_nil(rvals[rstart])) |
| 831 | rstart++; |
| 832 | } |
| 833 | } |
| 834 | /* from here on we don't have to worry about nil values */ |
| 835 | |
| 836 | while (lstart < lend && rstart < rend) { |
| 837 | v = rvals[rstart]; |
| 838 | |
| 839 | if (lscan < lend - lstart && lvals[lstart + lscan] < v) { |
| 840 | lstart = binsearch_int(NULL, 0, lvals, lstart + lscan, |
| 841 | lend - 1, v, 1, 0); |
| 842 | } else { |
| 843 | /* scan l for v */ |
| 844 | while (lstart < lend && lvals[lstart] < v) |
| 845 | lstart++; |
| 846 | } |
| 847 | if (lstart >= lend) { |
| 848 | /* nothing found */ |
| 849 | break; |
| 850 | } |
| 851 | |
| 852 | /* Here we determine the next value in l that we are |
| 853 | * going to try to match in r. We will also count the |
| 854 | * number of occurrences in l of that value. |
| 855 | * Afterwards, v points to the value and nl is the |
| 856 | * number of times it occurs. Also, lstart will |
| 857 | * point to the next value to be considered (ready for |
| 858 | * the next iteration). |
| 859 | * If there are many equal values in l (more than |
| 860 | * lscan), we will use binary search to find the end |
| 861 | * of the sequence. Obviously, we can do this only if |
| 862 | * l is actually sorted (lscan > 0). */ |
| 863 | nl = 1; /* we'll match (at least) one in l */ |
| 864 | nr = 0; /* maybe we won't match anything in r */ |
| 865 | v = lvals[lstart]; |
| 866 | if (l->tkey) { |
| 867 | /* if l is key, there is a single value */ |
| 868 | lstart++; |
| 869 | } else if (lscan < lend - lstart && |
| 870 | v == lvals[lstart + lscan]) { |
| 871 | /* lots of equal values: use binary search to |
| 872 | * find end */ |
| 873 | nl = binsearch_int(NULL, 0, lvals, lstart + lscan, |
| 874 | lend - 1, v, 1, 1); |
| 875 | nl -= lstart; |
| 876 | lstart += nl; |
| 877 | } else { |
| 878 | /* just scan */ |
| 879 | while (++lstart < lend && v == lvals[lstart]) |
| 880 | nl++; |
| 881 | } |
| 882 | /* lstart points one beyond the value we're |
| 883 | * going to match: ready for the next iteration. */ |
| 884 | |
| 885 | /* First we find the first value in r that is at |
| 886 | * least as large as v, then we find the first |
| 887 | * value in r that is larger than v. The difference |
| 888 | * is the number of values equal to v and is stored in |
| 889 | * nr. |
| 890 | * We will use binary search on r to find both ends of |
| 891 | * the sequence of values that are equal to v in case |
| 892 | * the position is "too far" (more than rscan |
| 893 | * away). */ |
| 894 | |
| 895 | /* first find the location of the first value in r |
| 896 | * that is >= v, then find the location of the first |
| 897 | * value in r that is > v; the difference is the |
| 898 | * number of values equal to v */ |
| 899 | |
| 900 | /* look ahead a little (rscan) in r to see whether |
| 901 | * we're better off doing a binary search */ |
| 902 | if (rscan < rend - rstart && rvals[rstart + rscan] < v) { |
| 903 | /* value too far away in r: use binary |
| 904 | * search */ |
| 905 | rstart = binsearch_int(NULL, 0, rvals, rstart + rscan, |
| 906 | rend - 1, v, 1, 0); |
| 907 | } else { |
| 908 | /* scan r for v */ |
| 909 | while (rstart < rend && rvals[rstart] < v) |
| 910 | rstart++; |
| 911 | } |
| 912 | if (rstart == rend) { |
| 913 | /* nothing found */ |
| 914 | break; |
| 915 | } |
| 916 | |
| 917 | /* now find the end of the sequence of equal values v */ |
| 918 | |
| 919 | /* if r is key, there is zero or one match, otherwise |
| 920 | * look ahead a little (rscan) in r to see whether |
| 921 | * we're better off doing a binary search */ |
| 922 | if (r->tkey) { |
| 923 | if (rstart < rend && v == rvals[rstart]) { |
| 924 | nr = 1; |
| 925 | rstart++; |
| 926 | } |
| 927 | } else if (rscan < rend - rstart && |
| 928 | v == rvals[rstart + rscan]) { |
| 929 | /* range too large: use binary search */ |
| 930 | nr = binsearch_int(NULL, 0, rvals, rstart + rscan, |
| 931 | rend - 1, v, 1, 1); |
| 932 | nr -= rstart; |
| 933 | rstart += nr; |
| 934 | } else { |
| 935 | /* scan r for end of range */ |
| 936 | while (rstart < rend && v == rvals[rstart]) { |
| 937 | nr++; |
| 938 | rstart++; |
| 939 | } |
| 940 | } |
| 941 | /* rstart points to first value > v or end of |
| 942 | * r, and nr is the number of values in r that |
| 943 | * are equal to v */ |
| 944 | if (nr == 0) { |
| 945 | /* no entries in r found */ |
| 946 | continue; |
| 947 | } |
| 948 | /* make space: nl values in l match nr values in r, so |
| 949 | * we need to add nl * nr values in the results */ |
| 950 | MAYBEEXTEND_NO_CAND(nl * nr); |
| 951 | |
| 952 | /* maintain properties */ |
| 953 | if (nl > 1) { |
| 954 | /* value occurs multiple times in l, so entry |
| 955 | * in r will be repeated multiple times: hence |
| 956 | * r2 is not key and not dense */ |
| 957 | if (r2) { |
| 958 | r2->tkey = false; |
| 959 | r2->tseqbase = oid_nil; |
| 960 | } |
| 961 | /* multiple different values will be inserted |
| 962 | * in r1 (always in order), so not reverse |
| 963 | * ordered anymore */ |
| 964 | r1->trevsorted = false; |
| 965 | } |
| 966 | if (nr > 1) { |
| 967 | /* value occurs multiple times in r, so entry |
| 968 | * in l will be repeated multiple times: hence |
| 969 | * r1 is not key and not dense */ |
| 970 | r1->tkey = false; |
| 971 | r1->tseqbase = oid_nil; |
| 972 | /* multiple different values will be inserted |
| 973 | * in r2 (in order), so not reverse ordered |
| 974 | * anymore */ |
| 975 | if (r2) { |
| 976 | r2->trevsorted = false; |
| 977 | if (nl > 1) { |
| 978 | /* multiple values in l match |
| 979 | * multiple values in r, so an |
| 980 | * ordered sequence will be |
| 981 | * inserted multiple times in |
| 982 | * r2, so r2 is not ordered |
| 983 | * anymore */ |
| 984 | r2->tsorted = false; |
| 985 | } |
| 986 | } |
| 987 | } |
| 988 | if (BATcount(r1) > 0) { |
| 989 | /* a new, higher value will be inserted into |
| 990 | * r1, so r1 is not reverse ordered anymore */ |
| 991 | r1->trevsorted = false; |
| 992 | /* a new higher value will be added to r2 */ |
| 993 | if (r2) { |
| 994 | r2->trevsorted = false; |
| 995 | } |
| 996 | if (BATtdense(r1) && |
| 997 | ((oid *) r1->theap.base)[r1->batCount - 1] + 1 != l->hseqbase + lstart - nl) { |
| 998 | r1->tseqbase = oid_nil; |
| 999 | } |
| 1000 | } |
| 1001 | |
| 1002 | if (r2 && |
| 1003 | BATcount(r2) > 0 && |
| 1004 | BATtdense(r2) && |
| 1005 | ((oid *) r2->theap.base)[r2->batCount - 1] + 1 != r->hseqbase + rstart - nr) { |
| 1006 | r2->tseqbase = oid_nil; |
| 1007 | } |
| 1008 | |
| 1009 | /* insert values */ |
| 1010 | lv = l->hseqbase + lstart - nl; |
| 1011 | for (i = 0; i < nl; i++) { |
| 1012 | BUN j; |
| 1013 | |
| 1014 | for (j = 0; j < nr; j++) { |
| 1015 | APPEND(r1, lv); |
| 1016 | } |
| 1017 | if (r2) { |
| 1018 | oid rv = r->hseqbase + rstart - nr; |
| 1019 | |
| 1020 | for (j = 0; j < nr; j++) { |
| 1021 | APPEND(r2, rv); |
| 1022 | rv++; |
| 1023 | } |
| 1024 | } |
| 1025 | lv++; |
| 1026 | } |
| 1027 | } |
| 1028 | /* also set other bits of heap to correct value to indicate size */ |
| 1029 | BATsetcount(r1, BATcount(r1)); |
| 1030 | if (r2) { |
| 1031 | BATsetcount(r2, BATcount(r2)); |
| 1032 | assert(BATcount(r1) == BATcount(r2)); |
| 1033 | } |
| 1034 | if (BATcount(r1) > 0) { |
| 1035 | if (BATtdense(r1)) |
| 1036 | r1->tseqbase = ((oid *) r1->theap.base)[0]; |
| 1037 | if (r2 && BATtdense(r2)) |
| 1038 | r2->tseqbase = ((oid *) r2->theap.base)[0]; |
| 1039 | } else { |
| 1040 | r1->tseqbase = 0; |
| 1041 | if (r2) { |
| 1042 | r2->tseqbase = 0; |
| 1043 | } |
| 1044 | } |
| 1045 | ALGODEBUG fprintf(stderr, "#%s: %s(l="ALGOBATFMT "," |
| 1046 | "r="ALGOBATFMT "," |
| 1047 | "nil_matches=%d)%s %s " |
| 1048 | "-> ("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 1049 | MT_thread_getname(), __func__, |
| 1050 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 1051 | nil_matches, |
| 1052 | swapped ? " swapped": "", reason, |
| 1053 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 1054 | GDKusec() - t0); |
| 1055 | |
| 1056 | return GDK_SUCCEED; |
| 1057 | |
| 1058 | bailout: |
| 1059 | BBPreclaim(r1); |
| 1060 | BBPreclaim(r2); |
| 1061 | return GDK_FAIL; |
| 1062 | } |
| 1063 | |
| 1064 | /* Implementation of mergejoin (see below) for the special case that |
| 1065 | * the values are of type lng, and some more conditions are met. */ |
| 1066 | static gdk_return |
| 1067 | mergejoin_lng(BAT **r1p, BAT **r2p, BAT *l, BAT *r, |
| 1068 | bool nil_matches, BUN estimate, lng t0, bool swapped, |
| 1069 | const char *reason) |
| 1070 | { |
| 1071 | BAT *r1, *r2; |
| 1072 | BUN lstart, lend, lcnt; |
| 1073 | BUN rstart, rend; |
| 1074 | BUN lscan, rscan; /* opportunistic scan window */ |
| 1075 | BUN maxsize; |
| 1076 | const lng *lvals, *rvals; |
| 1077 | lng v; |
| 1078 | BUN nl, nr; |
| 1079 | oid lv; |
| 1080 | BUN i; |
| 1081 | |
| 1082 | assert(ATOMtype(l->ttype) == ATOMtype(r->ttype)); |
| 1083 | assert(r->tsorted || r->trevsorted); |
| 1084 | |
| 1085 | lstart = rstart = 0; |
| 1086 | lend = BATcount(l); |
| 1087 | lcnt = lend - lstart; |
| 1088 | rend = BATcount(r); |
| 1089 | lvals = (const lng *) Tloc(l, 0); |
| 1090 | rvals = (const lng *) Tloc(r, 0); |
| 1091 | assert(!r->tvarsized || !r->ttype); |
| 1092 | |
| 1093 | /* basic properties will be adjusted if necessary later on, |
| 1094 | * they were initially set by joininitresults() */ |
| 1095 | |
| 1096 | if (lend == 0 || rend == 0) { |
| 1097 | /* there are no matches */ |
| 1098 | return nomatch(r1p, r2p, l, r, |
| 1099 | &(struct canditer) {.tpe = cand_dense, .ncand = lcnt,}, |
| 1100 | false, false, __func__, t0); |
| 1101 | } |
| 1102 | |
| 1103 | if ((maxsize = joininitresults(r1p, r2p, BATcount(l), BATcount(r), |
| 1104 | l->tkey, r->tkey, false, false, |
| 1105 | false, estimate)) == BUN_NONE) |
| 1106 | return GDK_FAIL; |
| 1107 | r1 = *r1p; |
| 1108 | r2 = r2p ? *r2p : NULL; |
| 1109 | |
| 1110 | /* determine opportunistic scan window for l and r */ |
| 1111 | for (nl = lend - lstart, lscan = 4; nl > 0; lscan++) |
| 1112 | nl >>= 1; |
| 1113 | for (nr = rend - rstart, rscan = 4; nr > 0; rscan++) |
| 1114 | nr >>= 1; |
| 1115 | |
| 1116 | if (!nil_matches) { |
| 1117 | /* skip over nils at the start of the columns */ |
| 1118 | if (lscan < lend - lstart && is_lng_nil(lvals[lstart + lscan])) { |
| 1119 | lstart = binsearch_lng(NULL, 0, lvals, lstart + lscan, |
| 1120 | lend - 1, lng_nil, 1, 1); |
| 1121 | } else { |
| 1122 | while (is_lng_nil(lvals[lstart])) |
| 1123 | lstart++; |
| 1124 | } |
| 1125 | if (rscan < rend - rstart && is_lng_nil(rvals[rstart + rscan])) { |
| 1126 | rstart = binsearch_lng(NULL, 0, rvals, rstart + rscan, |
| 1127 | rend - 1, lng_nil, 1, 1); |
| 1128 | } else { |
| 1129 | while (is_lng_nil(rvals[rstart])) |
| 1130 | rstart++; |
| 1131 | } |
| 1132 | } |
| 1133 | /* from here on we don't have to worry about nil values */ |
| 1134 | |
| 1135 | while (lstart < lend && rstart < rend) { |
| 1136 | v = rvals[rstart]; |
| 1137 | |
| 1138 | if (lscan < lend - lstart && lvals[lstart + lscan] < v) { |
| 1139 | lstart = binsearch_lng(NULL, 0, lvals, lstart + lscan, |
| 1140 | lend - 1, v, 1, 0); |
| 1141 | } else { |
| 1142 | /* scan l for v */ |
| 1143 | while (lstart < lend && lvals[lstart] < v) |
| 1144 | lstart++; |
| 1145 | } |
| 1146 | if (lstart >= lend) { |
| 1147 | /* nothing found */ |
| 1148 | break; |
| 1149 | } |
| 1150 | |
| 1151 | /* Here we determine the next value in l that we are |
| 1152 | * going to try to match in r. We will also count the |
| 1153 | * number of occurrences in l of that value. |
| 1154 | * Afterwards, v points to the value and nl is the |
| 1155 | * number of times it occurs. Also, lstart will |
| 1156 | * point to the next value to be considered (ready for |
| 1157 | * the next iteration). |
| 1158 | * If there are many equal values in l (more than |
| 1159 | * lscan), we will use binary search to find the end |
| 1160 | * of the sequence. Obviously, we can do this only if |
| 1161 | * l is actually sorted (lscan > 0). */ |
| 1162 | nl = 1; /* we'll match (at least) one in l */ |
| 1163 | nr = 0; /* maybe we won't match anything in r */ |
| 1164 | v = lvals[lstart]; |
| 1165 | if (l->tkey) { |
| 1166 | /* if l is key, there is a single value */ |
| 1167 | lstart++; |
| 1168 | } else if (lscan < lend - lstart && |
| 1169 | v == lvals[lstart + lscan]) { |
| 1170 | /* lots of equal values: use binary search to |
| 1171 | * find end */ |
| 1172 | nl = binsearch_lng(NULL, 0, lvals, lstart + lscan, |
| 1173 | lend - 1, v, 1, 1); |
| 1174 | nl -= lstart; |
| 1175 | lstart += nl; |
| 1176 | } else { |
| 1177 | /* just scan */ |
| 1178 | while (++lstart < lend && v == lvals[lstart]) |
| 1179 | nl++; |
| 1180 | } |
| 1181 | /* lstart points one beyond the value we're |
| 1182 | * going to match: ready for the next iteration. */ |
| 1183 | |
| 1184 | /* First we find the first value in r that is at |
| 1185 | * least as large as v, then we find the first |
| 1186 | * value in r that is larger than v. The difference |
| 1187 | * is the number of values equal to v and is stored in |
| 1188 | * nr. |
| 1189 | * We will use binary search on r to find both ends of |
| 1190 | * the sequence of values that are equal to v in case |
| 1191 | * the position is "too far" (more than rscan |
| 1192 | * away). */ |
| 1193 | |
| 1194 | /* first find the location of the first value in r |
| 1195 | * that is >= v, then find the location of the first |
| 1196 | * value in r that is > v; the difference is the |
| 1197 | * number of values equal to v */ |
| 1198 | |
| 1199 | /* look ahead a little (rscan) in r to see whether |
| 1200 | * we're better off doing a binary search */ |
| 1201 | if (rscan < rend - rstart && rvals[rstart + rscan] < v) { |
| 1202 | /* value too far away in r: use binary |
| 1203 | * search */ |
| 1204 | rstart = binsearch_lng(NULL, 0, rvals, rstart + rscan, |
| 1205 | rend - 1, v, 1, 0); |
| 1206 | } else { |
| 1207 | /* scan r for v */ |
| 1208 | while (rstart < rend && rvals[rstart] < v) |
| 1209 | rstart++; |
| 1210 | } |
| 1211 | if (rstart == rend) { |
| 1212 | /* nothing found */ |
| 1213 | break; |
| 1214 | } |
| 1215 | |
| 1216 | /* now find the end of the sequence of equal values v */ |
| 1217 | |
| 1218 | /* if r is key, there is zero or one match, otherwise |
| 1219 | * look ahead a little (rscan) in r to see whether |
| 1220 | * we're better off doing a binary search */ |
| 1221 | if (r->tkey) { |
| 1222 | if (rstart < rend && v == rvals[rstart]) { |
| 1223 | nr = 1; |
| 1224 | rstart++; |
| 1225 | } |
| 1226 | } else if (rscan < rend - rstart && |
| 1227 | v == rvals[rstart + rscan]) { |
| 1228 | /* range too large: use binary search */ |
| 1229 | nr = binsearch_lng(NULL, 0, rvals, rstart + rscan, |
| 1230 | rend - 1, v, 1, 1); |
| 1231 | nr -= rstart; |
| 1232 | rstart += nr; |
| 1233 | } else { |
| 1234 | /* scan r for end of range */ |
| 1235 | while (rstart < rend && v == rvals[rstart]) { |
| 1236 | nr++; |
| 1237 | rstart++; |
| 1238 | } |
| 1239 | } |
| 1240 | /* rstart points to first value > v or end of |
| 1241 | * r, and nr is the number of values in r that |
| 1242 | * are equal to v */ |
| 1243 | if (nr == 0) { |
| 1244 | /* no entries in r found */ |
| 1245 | continue; |
| 1246 | } |
| 1247 | /* make space: nl values in l match nr values in r, so |
| 1248 | * we need to add nl * nr values in the results */ |
| 1249 | MAYBEEXTEND_NO_CAND(nl * nr); |
| 1250 | |
| 1251 | /* maintain properties */ |
| 1252 | if (nl > 1) { |
| 1253 | /* value occurs multiple times in l, so entry |
| 1254 | * in r will be repeated multiple times: hence |
| 1255 | * r2 is not key and not dense */ |
| 1256 | if (r2) { |
| 1257 | r2->tkey = false; |
| 1258 | r2->tseqbase = oid_nil; |
| 1259 | } |
| 1260 | /* multiple different values will be inserted |
| 1261 | * in r1 (always in order), so not reverse |
| 1262 | * ordered anymore */ |
| 1263 | r1->trevsorted = false; |
| 1264 | } |
| 1265 | if (nr > 1) { |
| 1266 | /* value occurs multiple times in r, so entry |
| 1267 | * in l will be repeated multiple times: hence |
| 1268 | * r1 is not key and not dense */ |
| 1269 | r1->tkey = false; |
| 1270 | r1->tseqbase = oid_nil; |
| 1271 | /* multiple different values will be inserted |
| 1272 | * in r2 (in order), so not reverse ordered |
| 1273 | * anymore */ |
| 1274 | if (r2) { |
| 1275 | r2->trevsorted = false; |
| 1276 | if (nl > 1) { |
| 1277 | /* multiple values in l match |
| 1278 | * multiple values in r, so an |
| 1279 | * ordered sequence will be |
| 1280 | * inserted multiple times in |
| 1281 | * r2, so r2 is not ordered |
| 1282 | * anymore */ |
| 1283 | r2->tsorted = false; |
| 1284 | } |
| 1285 | } |
| 1286 | } |
| 1287 | if (BATcount(r1) > 0) { |
| 1288 | /* a new, higher value will be inserted into |
| 1289 | * r1, so r1 is not reverse ordered anymore */ |
| 1290 | r1->trevsorted = false; |
| 1291 | /* a new higher value will be added to r2 */ |
| 1292 | if (r2) { |
| 1293 | r2->trevsorted = false; |
| 1294 | } |
| 1295 | if (BATtdense(r1) && |
| 1296 | ((oid *) r1->theap.base)[r1->batCount - 1] + 1 != l->hseqbase + lstart - nl) { |
| 1297 | r1->tseqbase = oid_nil; |
| 1298 | } |
| 1299 | } |
| 1300 | |
| 1301 | if (r2 && |
| 1302 | BATcount(r2) > 0 && |
| 1303 | BATtdense(r2) && |
| 1304 | ((oid *) r2->theap.base)[r2->batCount - 1] + 1 != r->hseqbase + rstart - nr) { |
| 1305 | r2->tseqbase = oid_nil; |
| 1306 | } |
| 1307 | |
| 1308 | /* insert values */ |
| 1309 | lv = l->hseqbase + lstart - nl; |
| 1310 | for (i = 0; i < nl; i++) { |
| 1311 | BUN j; |
| 1312 | |
| 1313 | for (j = 0; j < nr; j++) { |
| 1314 | APPEND(r1, lv); |
| 1315 | } |
| 1316 | if (r2) { |
| 1317 | oid rv = r->hseqbase + rstart - nr; |
| 1318 | |
| 1319 | for (j = 0; j < nr; j++) { |
| 1320 | APPEND(r2, rv); |
| 1321 | rv++; |
| 1322 | } |
| 1323 | } |
| 1324 | lv++; |
| 1325 | } |
| 1326 | } |
| 1327 | /* also set other bits of heap to correct value to indicate size */ |
| 1328 | BATsetcount(r1, BATcount(r1)); |
| 1329 | if (r2) { |
| 1330 | BATsetcount(r2, BATcount(r2)); |
| 1331 | assert(BATcount(r1) == BATcount(r2)); |
| 1332 | } |
| 1333 | if (BATcount(r1) > 0) { |
| 1334 | if (BATtdense(r1)) |
| 1335 | r1->tseqbase = ((oid *) r1->theap.base)[0]; |
| 1336 | if (r2 && BATtdense(r2)) |
| 1337 | r2->tseqbase = ((oid *) r2->theap.base)[0]; |
| 1338 | } else { |
| 1339 | r1->tseqbase = 0; |
| 1340 | if (r2) { |
| 1341 | r2->tseqbase = 0; |
| 1342 | } |
| 1343 | } |
| 1344 | ALGODEBUG fprintf(stderr, "#%s: %s(l="ALGOBATFMT "," |
| 1345 | "r="ALGOBATFMT "," |
| 1346 | "nil_matches=%d)%s %s " |
| 1347 | "-> ("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 1348 | MT_thread_getname(), __func__, |
| 1349 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 1350 | nil_matches, |
| 1351 | swapped ? " swapped": "", reason, |
| 1352 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 1353 | GDKusec() - t0); |
| 1354 | |
| 1355 | return GDK_SUCCEED; |
| 1356 | |
| 1357 | bailout: |
| 1358 | BBPreclaim(r1); |
| 1359 | BBPreclaim(r2); |
| 1360 | return GDK_FAIL; |
| 1361 | } |
| 1362 | |
| 1363 | /* Implementation of mergejoin (see below) for the special case that |
| 1364 | * the values are of type oid, and the right-hand side is a candidate |
| 1365 | * list with exception, and some more conditions are met. */ |
| 1366 | static gdk_return |
| 1367 | mergejoin_cand(BAT **r1p, BAT **r2p, BAT *l, BAT *r, |
| 1368 | bool nil_matches, BUN estimate, lng t0, bool swapped, |
| 1369 | const char *reason) |
| 1370 | { |
| 1371 | BAT *r1, *r2; |
| 1372 | BUN lstart, lend, lcnt; |
| 1373 | struct canditer lci, rci; |
| 1374 | BUN lscan; /* opportunistic scan window */ |
| 1375 | BUN maxsize; |
| 1376 | const oid *lvals; |
| 1377 | oid v; |
| 1378 | BUN nl, nr; |
| 1379 | oid lv; |
| 1380 | BUN i; |
| 1381 | |
| 1382 | assert(ATOMtype(l->ttype) == ATOMtype(r->ttype)); |
| 1383 | |
| 1384 | lstart = 0; |
| 1385 | lend = BATcount(l); |
| 1386 | lcnt = lend - lstart; |
| 1387 | if (l->ttype == TYPE_void) { |
| 1388 | assert(!is_oid_nil(l->tseqbase)); |
| 1389 | lcnt = canditer_init(&lci, NULL, l); |
| 1390 | lvals = NULL; |
| 1391 | } else { |
| 1392 | lci = (struct canditer) {.tpe = cand_dense}; /* not used */ |
| 1393 | lvals = (const oid *) Tloc(l, 0); |
| 1394 | assert(lvals != NULL); |
| 1395 | } |
| 1396 | |
| 1397 | assert(r->ttype == TYPE_void && r->tvheap != NULL); |
| 1398 | canditer_init(&rci, NULL, r); |
| 1399 | |
| 1400 | /* basic properties will be adjusted if necessary later on, |
| 1401 | * they were initially set by joininitresults() */ |
| 1402 | |
| 1403 | if (lend == 0 || rci.ncand == 0) { |
| 1404 | /* there are no matches */ |
| 1405 | return nomatch(r1p, r2p, l, r, |
| 1406 | &(struct canditer) {.tpe = cand_dense, .ncand = lcnt,}, |
| 1407 | false, false, __func__, t0); |
| 1408 | } |
| 1409 | |
| 1410 | if ((maxsize = joininitresults(r1p, r2p, BATcount(l), BATcount(r), |
| 1411 | l->tkey, r->tkey, false, false, |
| 1412 | false, estimate)) == BUN_NONE) |
| 1413 | return GDK_FAIL; |
| 1414 | r1 = *r1p; |
| 1415 | r2 = r2p ? *r2p : NULL; |
| 1416 | |
| 1417 | /* determine opportunistic scan window for l and r */ |
| 1418 | for (nl = lend - lstart, lscan = 4; nl > 0; lscan++) |
| 1419 | nl >>= 1; |
| 1420 | |
| 1421 | if (!nil_matches) { |
| 1422 | /* skip over nils at the start of the columns */ |
| 1423 | if (lscan < lend - lstart && lvals && is_oid_nil(lvals[lstart + lscan])) { |
| 1424 | lstart = binsearch_oid(NULL, 0, lvals, lstart + lscan, |
| 1425 | lend - 1, oid_nil, 1, 1); |
| 1426 | } else if (lvals) { |
| 1427 | while (is_oid_nil(lvals[lstart])) |
| 1428 | lstart++; |
| 1429 | } /* else l is candidate list: no nils */ |
| 1430 | } |
| 1431 | /* from here on we don't have to worry about nil values */ |
| 1432 | |
| 1433 | while (lstart < lend && rci.next < rci.ncand) { |
| 1434 | v = canditer_peek(&rci); |
| 1435 | |
| 1436 | if (lvals) { |
| 1437 | if (lscan < lend - lstart && |
| 1438 | lvals[lstart + lscan] < v) { |
| 1439 | lstart = binsearch_oid(NULL, 0, lvals, |
| 1440 | lstart + lscan, |
| 1441 | lend - 1, v, 1, 0); |
| 1442 | } else { |
| 1443 | /* scan l for v */ |
| 1444 | while (lstart < lend && lvals[lstart] < v) |
| 1445 | lstart++; |
| 1446 | } |
| 1447 | } else { |
| 1448 | lstart = canditer_search(&lci, v, true); |
| 1449 | canditer_setidx(&lci, lstart); |
| 1450 | } |
| 1451 | if (lstart >= lend) { |
| 1452 | /* nothing found */ |
| 1453 | break; |
| 1454 | } |
| 1455 | |
| 1456 | /* Here we determine the next value in l that we are |
| 1457 | * going to try to match in r. We will also count the |
| 1458 | * number of occurrences in l of that value. |
| 1459 | * Afterwards, v points to the value and nl is the |
| 1460 | * number of times it occurs. Also, lstart will |
| 1461 | * point to the next value to be considered (ready for |
| 1462 | * the next iteration). |
| 1463 | * If there are many equal values in l (more than |
| 1464 | * lscan), we will use binary search to find the end |
| 1465 | * of the sequence. Obviously, we can do this only if |
| 1466 | * l is actually sorted (lscan > 0). */ |
| 1467 | nl = 1; /* we'll match (at least) one in l */ |
| 1468 | nr = 0; /* maybe we won't match anything in r */ |
| 1469 | v = lvals ? lvals[lstart] : canditer_next(&lci); |
| 1470 | if (l->tkey || lvals == NULL) { |
| 1471 | /* if l is key, there is a single value */ |
| 1472 | lstart++; |
| 1473 | } else if (lscan < lend - lstart && |
| 1474 | v == lvals[lstart + lscan]) { |
| 1475 | /* lots of equal values: use binary search to |
| 1476 | * find end */ |
| 1477 | nl = binsearch_oid(NULL, 0, lvals, lstart + lscan, |
| 1478 | lend - 1, v, 1, 1); |
| 1479 | nl -= lstart; |
| 1480 | lstart += nl; |
| 1481 | } else { |
| 1482 | /* just scan */ |
| 1483 | while (++lstart < lend && v == lvals[lstart]) |
| 1484 | nl++; |
| 1485 | } |
| 1486 | /* lstart points one beyond the value we're |
| 1487 | * going to match: ready for the next iteration. */ |
| 1488 | |
| 1489 | /* First we find the first value in r that is at |
| 1490 | * least as large as v, then we find the first |
| 1491 | * value in r that is larger than v. The difference |
| 1492 | * is the number of values equal to v and is stored in |
| 1493 | * nr. |
| 1494 | * We will use binary search on r to find both ends of |
| 1495 | * the sequence of values that are equal to v in case |
| 1496 | * the position is "too far" (more than rscan |
| 1497 | * away). */ |
| 1498 | |
| 1499 | /* first find the location of the first value in r |
| 1500 | * that is >= v, then find the location of the first |
| 1501 | * value in r that is > v; the difference is the |
| 1502 | * number of values equal to v */ |
| 1503 | nr = canditer_search(&rci, v, true); |
| 1504 | canditer_setidx(&rci, nr); |
| 1505 | if (nr == rci.ncand) { |
| 1506 | /* nothing found */ |
| 1507 | break; |
| 1508 | } |
| 1509 | |
| 1510 | /* now find the end of the sequence of equal values v */ |
| 1511 | |
| 1512 | /* if r is key, there is zero or one match, otherwise |
| 1513 | * look ahead a little (rscan) in r to see whether |
| 1514 | * we're better off doing a binary search */ |
| 1515 | if (canditer_peek(&rci) == v) { |
| 1516 | nr = 1; |
| 1517 | canditer_next(&rci); |
| 1518 | } else { |
| 1519 | /* rci points to first value > v or end of |
| 1520 | * r, and nr is the number of values in r that |
| 1521 | * are equal to v */ |
| 1522 | /* no entries in r found */ |
| 1523 | continue; |
| 1524 | } |
| 1525 | /* make space: nl values in l match nr values in r, so |
| 1526 | * we need to add nl * nr values in the results */ |
| 1527 | MAYBEEXTEND_NO_CAND(nl * nr); |
| 1528 | |
| 1529 | /* maintain properties */ |
| 1530 | if (nl > 1) { |
| 1531 | /* value occurs multiple times in l, so entry |
| 1532 | * in r will be repeated multiple times: hence |
| 1533 | * r2 is not key and not dense */ |
| 1534 | if (r2) { |
| 1535 | r2->tkey = false; |
| 1536 | r2->tseqbase = oid_nil; |
| 1537 | } |
| 1538 | /* multiple different values will be inserted |
| 1539 | * in r1 (always in order), so not reverse |
| 1540 | * ordered anymore */ |
| 1541 | r1->trevsorted = false; |
| 1542 | } |
| 1543 | if (nr > 1) { |
| 1544 | /* value occurs multiple times in r, so entry |
| 1545 | * in l will be repeated multiple times: hence |
| 1546 | * r1 is not key and not dense */ |
| 1547 | r1->tkey = false; |
| 1548 | r1->tseqbase = oid_nil; |
| 1549 | /* multiple different values will be inserted |
| 1550 | * in r2 (in order), so not reverse ordered |
| 1551 | * anymore */ |
| 1552 | if (r2) { |
| 1553 | r2->trevsorted = false; |
| 1554 | if (nl > 1) { |
| 1555 | /* multiple values in l match |
| 1556 | * multiple values in r, so an |
| 1557 | * ordered sequence will be |
| 1558 | * inserted multiple times in |
| 1559 | * r2, so r2 is not ordered |
| 1560 | * anymore */ |
| 1561 | r2->tsorted = false; |
| 1562 | } |
| 1563 | } |
| 1564 | } |
| 1565 | if (BATcount(r1) > 0) { |
| 1566 | /* a new, higher value will be inserted into |
| 1567 | * r1, so r1 is not reverse ordered anymore */ |
| 1568 | r1->trevsorted = false; |
| 1569 | /* a new higher value will be added to r2 */ |
| 1570 | if (r2) { |
| 1571 | r2->trevsorted = false; |
| 1572 | } |
| 1573 | if (BATtdense(r1) && |
| 1574 | ((oid *) r1->theap.base)[r1->batCount - 1] + 1 != l->hseqbase + lstart - nl) { |
| 1575 | r1->tseqbase = oid_nil; |
| 1576 | } |
| 1577 | } |
| 1578 | |
| 1579 | if (r2 && |
| 1580 | BATcount(r2) > 0 && |
| 1581 | BATtdense(r2) && |
| 1582 | ((oid *) r2->theap.base)[r2->batCount - 1] + 1 != r->hseqbase + rci.next - nr) { |
| 1583 | r2->tseqbase = oid_nil; |
| 1584 | } |
| 1585 | |
| 1586 | /* insert values */ |
| 1587 | lv = l->hseqbase + lstart - nl; |
| 1588 | for (i = 0; i < nl; i++) { |
| 1589 | BUN j; |
| 1590 | |
| 1591 | for (j = 0; j < nr; j++) { |
| 1592 | APPEND(r1, lv); |
| 1593 | } |
| 1594 | if (r2) { |
| 1595 | oid rv = r->hseqbase + rci.next - nr; |
| 1596 | |
| 1597 | for (j = 0; j < nr; j++) { |
| 1598 | APPEND(r2, rv); |
| 1599 | rv++; |
| 1600 | } |
| 1601 | } |
| 1602 | lv++; |
| 1603 | } |
| 1604 | } |
| 1605 | /* also set other bits of heap to correct value to indicate size */ |
| 1606 | BATsetcount(r1, BATcount(r1)); |
| 1607 | if (r2) { |
| 1608 | BATsetcount(r2, BATcount(r2)); |
| 1609 | assert(BATcount(r1) == BATcount(r2)); |
| 1610 | } |
| 1611 | if (BATcount(r1) > 0) { |
| 1612 | if (BATtdense(r1)) |
| 1613 | r1->tseqbase = ((oid *) r1->theap.base)[0]; |
| 1614 | if (r2 && BATtdense(r2)) |
| 1615 | r2->tseqbase = ((oid *) r2->theap.base)[0]; |
| 1616 | } else { |
| 1617 | r1->tseqbase = 0; |
| 1618 | if (r2) { |
| 1619 | r2->tseqbase = 0; |
| 1620 | } |
| 1621 | } |
| 1622 | ALGODEBUG fprintf(stderr, "#%s: %s(l="ALGOBATFMT "," |
| 1623 | "r="ALGOBATFMT "," |
| 1624 | "nil_matches=%d)%s %s " |
| 1625 | "-> ("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 1626 | MT_thread_getname(), __func__, |
| 1627 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 1628 | nil_matches, |
| 1629 | swapped ? " swapped": "", reason, |
| 1630 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 1631 | GDKusec() - t0); |
| 1632 | |
| 1633 | return GDK_SUCCEED; |
| 1634 | |
| 1635 | bailout: |
| 1636 | BBPreclaim(r1); |
| 1637 | BBPreclaim(r2); |
| 1638 | return GDK_FAIL; |
| 1639 | } |
| 1640 | |
| 1641 | /* Perform a "merge" join on l and r (if both are sorted) with |
| 1642 | * optional candidate lists, or join using binary search on r if l is |
| 1643 | * not sorted. The return BATs have already been created by the |
| 1644 | * caller. |
| 1645 | * |
| 1646 | * If nil_matches is set, nil values are treated as ordinary values |
| 1647 | * that can match; otherwise nil values never match. |
| 1648 | * |
| 1649 | * If nil_on_miss is set, a nil value is returned in r2 if there is no |
| 1650 | * match in r for a particular value in l (left outer join). |
| 1651 | * |
| 1652 | * If semi is set, only a single set of values in r1/r2 is returned if |
| 1653 | * there is a match of l in r, no matter how many matches there are in |
| 1654 | * r; otherwise all matches are returned. |
| 1655 | * |
| 1656 | * t0 and swapped are only for debugging (ALGOMASK set in GDKdebug). |
| 1657 | */ |
| 1658 | static gdk_return |
| 1659 | mergejoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, |
| 1660 | struct canditer *restrict lci, struct canditer *restrict rci, |
| 1661 | bool nil_matches, bool nil_on_miss, bool semi, bool only_misses, |
| 1662 | bool not_in, BUN estimate, lng t0, bool swapped, const char *reason) |
| 1663 | { |
| 1664 | /* [lr]scan determine how far we look ahead in l/r in order to |
| 1665 | * decide whether we want to do a binary search or a scan */ |
| 1666 | BUN lscan, rscan; |
| 1667 | const void *lvals, *rvals; /* the values of l/r (NULL if dense) */ |
| 1668 | const char *lvars, *rvars; /* the indirect values (NULL if fixed size) */ |
| 1669 | int lwidth, rwidth; /* width of the values */ |
| 1670 | const void *nil = ATOMnilptr(l->ttype); |
| 1671 | int (*cmp)(const void *, const void *) = ATOMcompare(l->ttype); |
| 1672 | const void *v; /* points to value under consideration */ |
| 1673 | const void *prev = NULL; |
| 1674 | BUN nl, nr; |
| 1675 | bool insert_nil; |
| 1676 | /* equal_order is set if we can scan both BATs in the same |
| 1677 | * order, so when both are sorted or both are reverse sorted |
| 1678 | * -- important to know in order to skip over values; if l is |
| 1679 | * not sorted, this must be set to true and we will always do a |
| 1680 | * binary search on all of r */ |
| 1681 | bool equal_order; |
| 1682 | /* [lr]ordering is either 1 or -1 depending on the order of |
| 1683 | * l/r: it determines the comparison function used */ |
| 1684 | int lordering, rordering; |
| 1685 | oid lv; |
| 1686 | BUN i, j; /* counters */ |
| 1687 | bool lskipped = false; /* whether we skipped values in l */ |
| 1688 | oid lval = oid_nil, rval = oid_nil; /* temporary space to point v to */ |
| 1689 | struct canditer llci, rrci; |
| 1690 | |
| 1691 | if (sl == NULL && sr == NULL && !nil_on_miss && |
| 1692 | !semi && !only_misses && !not_in && |
| 1693 | l->tsorted && r->tsorted) { |
| 1694 | /* special cases with far fewer options */ |
| 1695 | if (r->ttype == TYPE_void && r->tvheap) |
| 1696 | return mergejoin_cand(r1p, r2p, l, r, nil_matches, |
| 1697 | estimate, t0, swapped, __func__); |
| 1698 | switch (ATOMbasetype(l->ttype)) { |
| 1699 | case TYPE_int: |
| 1700 | return mergejoin_int(r1p, r2p, l, r, nil_matches, |
| 1701 | estimate, t0, swapped, __func__); |
| 1702 | case TYPE_lng: |
| 1703 | return mergejoin_lng(r1p, r2p, l, r, nil_matches, |
| 1704 | estimate, t0, swapped, __func__); |
| 1705 | } |
| 1706 | } |
| 1707 | |
| 1708 | assert(ATOMtype(l->ttype) == ATOMtype(r->ttype)); |
| 1709 | assert(r->tsorted || r->trevsorted); |
| 1710 | assert(sl == NULL || sl->tsorted); |
| 1711 | assert(sr == NULL || sr->tsorted); |
| 1712 | |
| 1713 | if (BATtvoid(l)) { |
| 1714 | /* l->ttype == TYPE_void && is_oid_nil(l->tseqbase) is |
| 1715 | * handled by selectjoin */ |
| 1716 | assert(!is_oid_nil(l->tseqbase)); |
| 1717 | canditer_init(&llci, NULL, l); |
| 1718 | lvals = NULL; |
| 1719 | } else { |
| 1720 | lvals = Tloc(l, 0); |
| 1721 | llci = (struct canditer) {.tpe = cand_dense}; /* not used */ |
| 1722 | } |
| 1723 | rrci = (struct canditer) {.tpe = cand_dense}; |
| 1724 | if (BATtvoid(r)) { |
| 1725 | if (!is_oid_nil(r->tseqbase)) |
| 1726 | canditer_init(&rrci, NULL, r); |
| 1727 | rvals = NULL; |
| 1728 | } else { |
| 1729 | rvals = Tloc(r, 0); |
| 1730 | } |
| 1731 | if (l->tvarsized && l->ttype) { |
| 1732 | assert(r->tvarsized && r->ttype); |
| 1733 | lvars = l->tvheap->base; |
| 1734 | rvars = r->tvheap->base; |
| 1735 | } else { |
| 1736 | assert(!r->tvarsized || !r->ttype); |
| 1737 | lvars = rvars = NULL; |
| 1738 | } |
| 1739 | lwidth = l->twidth; |
| 1740 | rwidth = r->twidth; |
| 1741 | |
| 1742 | /* basic properties will be adjusted if necessary later on, |
| 1743 | * they were initially set by joininitresults() */ |
| 1744 | |
| 1745 | if (not_in && rci->ncand > 0 && !r->tnonil && |
| 1746 | ((BATtvoid(l) && l->tseqbase == oid_nil) || |
| 1747 | (BATtvoid(r) && r->tseqbase == oid_nil) || |
| 1748 | (rvals && cmp(nil, VALUE(r, (r->tsorted ? rci->seq : canditer_last(rci)) - r->hseqbase)) == 0))) |
| 1749 | return nomatch(r1p, r2p, l, r, lci, false, false, |
| 1750 | "mergejoin", t0); |
| 1751 | |
| 1752 | if (lci->ncand == 0 || |
| 1753 | rci->ncand == 0 || |
| 1754 | (!nil_matches && |
| 1755 | ((l->ttype == TYPE_void && is_oid_nil(l->tseqbase)) || |
| 1756 | (r->ttype == TYPE_void && is_oid_nil(r->tseqbase)))) || |
| 1757 | (l->ttype == TYPE_void && is_oid_nil(l->tseqbase) && |
| 1758 | (r->tnonil || |
| 1759 | (r->ttype == TYPE_void && !is_oid_nil(r->tseqbase)))) || |
| 1760 | (r->ttype == TYPE_void && is_oid_nil(r->tseqbase) && |
| 1761 | (l->tnonil || |
| 1762 | (l->ttype == TYPE_void && !is_oid_nil(l->tseqbase))))) { |
| 1763 | /* there are no matches */ |
| 1764 | return nomatch(r1p, r2p, l, r, lci, nil_on_miss, only_misses, |
| 1765 | "mergejoin", t0); |
| 1766 | } |
| 1767 | |
| 1768 | BUN maxsize = joininitresults(r1p, r2p, lci->ncand, rci->ncand, |
| 1769 | l->tkey, r->tkey, semi, nil_on_miss, |
| 1770 | only_misses, estimate); |
| 1771 | if (maxsize == BUN_NONE) |
| 1772 | return GDK_FAIL; |
| 1773 | BAT *r1 = *r1p; |
| 1774 | BAT *r2 = r2p ? *r2p : NULL; |
| 1775 | |
| 1776 | if (l->tsorted || l->trevsorted) { |
| 1777 | /* determine opportunistic scan window for l */ |
| 1778 | for (nl = lci->ncand, lscan = 4; nl > 0; lscan++) |
| 1779 | nl >>= 1; |
| 1780 | equal_order = (l->tsorted && r->tsorted) || |
| 1781 | (l->trevsorted && r->trevsorted && |
| 1782 | !BATtvoid(l) && !BATtvoid(r)); |
| 1783 | lordering = l->tsorted && (r->tsorted || !equal_order) ? 1 : -1; |
| 1784 | rordering = equal_order ? lordering : -lordering; |
| 1785 | } else { |
| 1786 | /* if l not sorted, we will always use binary search |
| 1787 | * on r */ |
| 1788 | assert(!BATtvoid(l)); /* void is always sorted */ |
| 1789 | lscan = 0; |
| 1790 | equal_order = true; |
| 1791 | lordering = 1; |
| 1792 | rordering = r->tsorted ? 1 : -1; |
| 1793 | } |
| 1794 | /* determine opportunistic scan window for r; if l is not |
| 1795 | * sorted this is only used to find range of equal values */ |
| 1796 | for (nl = rci->ncand, rscan = 4; nl > 0; rscan++) |
| 1797 | nl >>= 1; |
| 1798 | |
| 1799 | if (!equal_order) { |
| 1800 | /* we go through r backwards */ |
| 1801 | canditer_setidx(rci, rci->ncand); |
| 1802 | } |
| 1803 | /* At this point the various variables that help us through |
| 1804 | * the algorithm have been set. The table explains them. The |
| 1805 | * first two columns are the inputs, the next three columns |
| 1806 | * are the variables, the final two columns indicate how the |
| 1807 | * variables can be used. |
| 1808 | * |
| 1809 | * l/r sl/sr | vals cand off | result value being matched |
| 1810 | * -------------+-----------------+---------------------------------- |
| 1811 | * dense NULL | NULL NULL set | i off==nil?nil:i+off |
| 1812 | * dense dense | NULL NULL set | i off==nil?nil:i+off |
| 1813 | * dense set | NULL set set | cand[i] off==nil?nil:cand[i]+off |
| 1814 | * set NULL | set NULL 0 | i vals[i] |
| 1815 | * set dense | set NULL 0 | i vals[i] |
| 1816 | * set set | set set 0 | cand[i] vals[cand[i]] |
| 1817 | * |
| 1818 | * If {l,r}off is lng_nil, all values in the corresponding bat |
| 1819 | * are oid_nil because the bat has type VOID and the tseqbase |
| 1820 | * is nil. |
| 1821 | */ |
| 1822 | |
| 1823 | /* Before we start adding values to r1 and r2, the properties |
| 1824 | * are as follows: |
| 1825 | * tseqbase - 0 |
| 1826 | * tkey - true |
| 1827 | * tsorted - true |
| 1828 | * trevsorted - true |
| 1829 | * tnil - false |
| 1830 | * tnonil - true |
| 1831 | * We will modify these as we go along. |
| 1832 | */ |
| 1833 | while (lci->next < lci->ncand) { |
| 1834 | if (lscan == 0) { |
| 1835 | /* always search r completely */ |
| 1836 | assert(equal_order); |
| 1837 | canditer_reset(rci); |
| 1838 | } else { |
| 1839 | /* If l is sorted (lscan > 0), we look at the |
| 1840 | * next value in r to see whether we can jump |
| 1841 | * over a large section of l using binary |
| 1842 | * search. We do this by looking ahead in l |
| 1843 | * (lscan far, to be precise) and seeing if |
| 1844 | * the value there is still too "small" |
| 1845 | * (definition depends on sort order of l). |
| 1846 | * If it is, we use binary search on l, |
| 1847 | * otherwise we scan l for the next position |
| 1848 | * with a value greater than or equal to the |
| 1849 | * value in r. |
| 1850 | * The next value to match in r is the first |
| 1851 | * if equal_order is set, the last |
| 1852 | * otherwise. |
| 1853 | * When skipping over values in l, we count |
| 1854 | * how many we skip in nlx. We need this in |
| 1855 | * case only_misses or nil_on_miss is set, and |
| 1856 | * to properly set the dense property in the |
| 1857 | * first output BAT. */ |
| 1858 | BUN nlx = 0; /* number of non-matching values in l */ |
| 1859 | |
| 1860 | if (equal_order) { |
| 1861 | if (rci->next == rci->ncand) |
| 1862 | v = NULL; /* no more values */ |
| 1863 | else |
| 1864 | v = VALUE(r, canditer_peek(rci) - r->hseqbase); |
| 1865 | } else { |
| 1866 | if (rci->next == 0) |
| 1867 | v = NULL; /* no more values */ |
| 1868 | else |
| 1869 | v = VALUE(r, canditer_peekprev(rci) - r->hseqbase); |
| 1870 | } |
| 1871 | /* here, v points to next value in r, or if |
| 1872 | * we're at the end of r, v is NULL */ |
| 1873 | if (v == NULL) { |
| 1874 | nlx = lci->ncand - lci->next; |
| 1875 | } else { |
| 1876 | if (lscan < lci->ncand - lci->next) { |
| 1877 | lv = canditer_idx(lci, lci->next + lscan); |
| 1878 | lv -= l->hseqbase; |
| 1879 | if (lvals) { |
| 1880 | if (lordering * cmp(VALUE(l, lv), v) < 0) { |
| 1881 | nlx = binsearch(NULL, 0, l->ttype, lvals, lvars, lwidth, lv, BATcount(l), v, lordering, 0); |
| 1882 | nlx = canditer_search(lci, nlx + l->hseqbase, true); |
| 1883 | nlx -= lci->next; |
| 1884 | } |
| 1885 | } else { |
| 1886 | assert(lordering == 1); |
| 1887 | if (canditer_idx(&llci, lv) < *(const oid *)v) { |
| 1888 | nlx = canditer_search(&llci, *(const oid *)v, true); |
| 1889 | nlx = canditer_search(lci, nlx + l->hseqbase, true); |
| 1890 | nlx -= lci->next; |
| 1891 | } |
| 1892 | } |
| 1893 | if (lci->next + nlx == lci->ncand) |
| 1894 | v = NULL; |
| 1895 | } |
| 1896 | } |
| 1897 | if (nlx > 0) { |
| 1898 | if (only_misses) { |
| 1899 | MAYBEEXTEND(nlx, lci); |
| 1900 | lskipped |= nlx > 1 && lci->tpe != cand_dense; |
| 1901 | while (nlx > 0) { |
| 1902 | APPEND(r1, canditer_next(lci)); |
| 1903 | nlx--; |
| 1904 | } |
| 1905 | if (lskipped) |
| 1906 | r1->tseqbase = oid_nil; |
| 1907 | if (r1->trevsorted && BATcount(r1) > 1) |
| 1908 | r1->trevsorted = false; |
| 1909 | } else if (nil_on_miss) { |
| 1910 | if (r2->tnonil) { |
| 1911 | r2->tnil = true; |
| 1912 | r2->tnonil = false; |
| 1913 | r2->tseqbase = oid_nil; |
| 1914 | r2->tsorted = false; |
| 1915 | r2->trevsorted = false; |
| 1916 | r2->tkey = false; |
| 1917 | } |
| 1918 | MAYBEEXTEND(nlx, lci); |
| 1919 | lskipped |= nlx > 1 && lci->tpe != cand_dense; |
| 1920 | while (nlx > 0) { |
| 1921 | APPEND(r1, canditer_next(lci)); |
| 1922 | APPEND(r2, oid_nil); |
| 1923 | nlx--; |
| 1924 | } |
| 1925 | if (lskipped) |
| 1926 | r1->tseqbase = oid_nil; |
| 1927 | if (r1->trevsorted && BATcount(r1) > 1) |
| 1928 | r1->trevsorted = false; |
| 1929 | } else { |
| 1930 | lskipped = BATcount(r1) > 0; |
| 1931 | canditer_setidx(lci, lci->next + nlx); |
| 1932 | } |
| 1933 | } |
| 1934 | if (v == NULL) { |
| 1935 | /* we have exhausted the inputs */ |
| 1936 | break; |
| 1937 | } |
| 1938 | } |
| 1939 | |
| 1940 | /* Here we determine the next value in l that we are |
| 1941 | * going to try to match in r. We will also count the |
| 1942 | * number of occurrences in l of that value. |
| 1943 | * Afterwards, v points to the value and nl is the |
| 1944 | * number of times it occurs. Also, lci will point to |
| 1945 | * the next value to be considered (ready for the next |
| 1946 | * iteration). |
| 1947 | * If there are many equal values in l (more than |
| 1948 | * lscan), we will use binary search to find the end |
| 1949 | * of the sequence. Obviously, we can do this only if |
| 1950 | * l is actually sorted (lscan > 0). */ |
| 1951 | nl = 1; /* we'll match (at least) one in l */ |
| 1952 | nr = 0; /* maybe we won't match anything in r */ |
| 1953 | v = VALUE(l, canditer_peek(lci) - l->hseqbase); |
| 1954 | if (l->tkey) { |
| 1955 | /* if l is key, there is a single value */ |
| 1956 | } else if (lscan > 0 && |
| 1957 | lscan < lci->ncand - lci->next && |
| 1958 | cmp(v, VALUE(l, canditer_idx(lci, lci->next + lscan) - l->hseqbase)) == 0) { |
| 1959 | /* lots of equal values: use binary search to |
| 1960 | * find end */ |
| 1961 | assert(lvals != NULL); |
| 1962 | nl = binsearch(NULL, 0, |
| 1963 | l->ttype, lvals, lvars, |
| 1964 | lwidth, lci->next + lscan, |
| 1965 | BATcount(l), |
| 1966 | v, lordering, 1); |
| 1967 | nl = canditer_search(lci, nl + l->hseqbase, true); |
| 1968 | nl -= lci->next; |
| 1969 | } else { |
| 1970 | struct canditer ci = *lci; /* work on copy */ |
| 1971 | nl = 0; /* it will be incremented again */ |
| 1972 | do { |
| 1973 | canditer_next(&ci); |
| 1974 | nl++; |
| 1975 | } while (ci.next < ci.ncand && |
| 1976 | cmp(v, VALUE(l, canditer_peek(&ci) - l->hseqbase)) == 0); |
| 1977 | } |
| 1978 | /* lci->next + nl is the position for the next iteration */ |
| 1979 | |
| 1980 | if ((!nil_matches || not_in) && !l->tnonil && cmp(v, nil) == 0) { |
| 1981 | if (not_in) { |
| 1982 | /* just skip the whole thing: nils |
| 1983 | * don't cause any output */ |
| 1984 | canditer_setidx(lci, lci->next + nl); |
| 1985 | continue; |
| 1986 | } |
| 1987 | /* v is nil and nils don't match anything, set |
| 1988 | * to NULL to indicate nil */ |
| 1989 | v = NULL; |
| 1990 | } |
| 1991 | |
| 1992 | /* First we find the "first" value in r that is "at |
| 1993 | * least as large" as v, then we find the "first" |
| 1994 | * value in r that is "larger" than v. The difference |
| 1995 | * is the number of values equal to v and is stored in |
| 1996 | * nr. The definitions of "larger" and "first" depend |
| 1997 | * on the orderings of l and r. If equal_order is |
| 1998 | * set, we go through r from low to high (this |
| 1999 | * includes the case that l is not sorted); otherwise |
| 2000 | * we go through r from high to low. |
| 2001 | * In either case, we will use binary search on r to |
| 2002 | * find both ends of the sequence of values that are |
| 2003 | * equal to v in case the position is "too far" (more |
| 2004 | * than rscan away). */ |
| 2005 | if (v == NULL) { |
| 2006 | nr = 0; /* nils don't match anything */ |
| 2007 | } else if (r->ttype == TYPE_void && is_oid_nil(r->tseqbase)) { |
| 2008 | if (is_oid_nil(*(const oid *) v)) { |
| 2009 | /* all values in r match */ |
| 2010 | nr = rci->ncand; |
| 2011 | } else { |
| 2012 | /* no value in r matches */ |
| 2013 | nr = 0; |
| 2014 | } |
| 2015 | /* in either case, we're done after this */ |
| 2016 | canditer_setidx(rci, equal_order ? rci->ncand : 0); |
| 2017 | } else if (equal_order) { |
| 2018 | /* first find the location of the first value |
| 2019 | * in r that is >= v, then find the location |
| 2020 | * of the first value in r that is > v; the |
| 2021 | * difference is the number of values equal |
| 2022 | * v; we change rci */ |
| 2023 | |
| 2024 | /* look ahead a little (rscan) in r to |
| 2025 | * see whether we're better off doing |
| 2026 | * a binary search */ |
| 2027 | if (rvals) { |
| 2028 | if (rscan < rci->ncand - rci->next && |
| 2029 | rordering * cmp(v, VALUE(r, canditer_idx(rci, rci->next + rscan) - r->hseqbase)) > 0) { |
| 2030 | /* value too far away in r: |
| 2031 | * use binary search */ |
| 2032 | lv = binsearch(NULL, 0, r->ttype, rvals, rvars, rwidth, rci->next + rscan, BATcount(r), v, rordering, 0); |
| 2033 | lv = canditer_search(rci, lv + r->hseqbase, true); |
| 2034 | canditer_setidx(rci, lv); |
| 2035 | } else { |
| 2036 | /* scan r for v */ |
| 2037 | while (rci->next < rci->ncand) { |
| 2038 | if (rordering * cmp(v, VALUE(r, canditer_peek(rci) - r->hseqbase)) <= 0) |
| 2039 | break; |
| 2040 | canditer_next(rci); |
| 2041 | } |
| 2042 | } |
| 2043 | if (rci->next < rci->ncand && |
| 2044 | cmp(v, VALUE(r, canditer_peek(rci) - r->hseqbase)) == 0) { |
| 2045 | /* if we found an equal value, |
| 2046 | * look for the last equal |
| 2047 | * value */ |
| 2048 | if (r->tkey) { |
| 2049 | /* r is key, there can |
| 2050 | * only be a single |
| 2051 | * equal value */ |
| 2052 | nr = 1; |
| 2053 | canditer_next(rci); |
| 2054 | } else if (rscan < rci->ncand - rci->next && |
| 2055 | cmp(v, VALUE(r, canditer_idx(rci, rci->next + rscan) - r->hseqbase)) == 0) { |
| 2056 | /* many equal values: |
| 2057 | * use binary search |
| 2058 | * to find the end */ |
| 2059 | nr = binsearch(NULL, 0, r->ttype, rvals, rvars, rwidth, rci->next + rscan, BATcount(r), v, rordering, 1); |
| 2060 | nr = canditer_search(rci, nr + r->hseqbase, true); |
| 2061 | nr -= rci->next; |
| 2062 | canditer_setidx(rci, rci->next + nr); |
| 2063 | } else { |
| 2064 | /* scan r for end of |
| 2065 | * range */ |
| 2066 | do { |
| 2067 | nr++; |
| 2068 | canditer_next(rci); |
| 2069 | } while (rci->next < rci->ncand && |
| 2070 | cmp(v, VALUE(r, canditer_peek(rci) - r->hseqbase)) == 0); |
| 2071 | } |
| 2072 | } |
| 2073 | } else { |
| 2074 | assert(rordering == 1); |
| 2075 | rval = canditer_search(&rrci, *(const oid*)v, true) + r->hseqbase; |
| 2076 | lv = canditer_search(rci, rval, true); |
| 2077 | canditer_setidx(rci, lv); |
| 2078 | nr = (canditer_idx(&rrci, canditer_peek(rci) - r->hseqbase) == *(oid*)v); |
| 2079 | if (nr == 1) |
| 2080 | canditer_next(rci); |
| 2081 | } |
| 2082 | /* rci points to first value > v or end of r, |
| 2083 | * and nr is the number of values in r that |
| 2084 | * are equal to v */ |
| 2085 | } else { |
| 2086 | /* first find the location of the first value |
| 2087 | * in r that is > v, then find the location |
| 2088 | * of the first value in r that is >= v; the |
| 2089 | * difference is the number of values equal |
| 2090 | * v; we change rci */ |
| 2091 | |
| 2092 | /* look back from the end a little |
| 2093 | * (rscan) in r to see whether we're |
| 2094 | * better off doing a binary search */ |
| 2095 | if (rvals) { |
| 2096 | if (rci->next > rscan && |
| 2097 | rordering * cmp(v, VALUE(r, canditer_idx(rci, rci->next - rscan) - r->hseqbase)) < 0) { |
| 2098 | /* value too far away |
| 2099 | * in r: use binary |
| 2100 | * search */ |
| 2101 | lv = binsearch(NULL, 0, r->ttype, rvals, rvars, rwidth, 0, rci->next - rscan, v, rordering, 1); |
| 2102 | lv = canditer_search(rci, lv + r->hseqbase, true); |
| 2103 | canditer_setidx(rci, lv); |
| 2104 | } else { |
| 2105 | /* scan r for v */ |
| 2106 | while (rci->next > 0 && |
| 2107 | rordering * cmp(v, VALUE(r, canditer_peekprev(rci) - r->hseqbase)) < 0) |
| 2108 | canditer_prev(rci); |
| 2109 | } |
| 2110 | if (rci->next > 0 && |
| 2111 | cmp(v, VALUE(r, canditer_peekprev(rci) - r->hseqbase)) == 0) { |
| 2112 | /* if we found an equal value, |
| 2113 | * look for the last equal |
| 2114 | * value */ |
| 2115 | if (r->tkey) { |
| 2116 | /* r is key, there can only be a single equal value */ |
| 2117 | nr = 1; |
| 2118 | canditer_prev(rci); |
| 2119 | } else if (rci->next > rscan && |
| 2120 | cmp(v, VALUE(r, canditer_idx(rci, rci->next - rscan) - r->hseqbase)) == 0) { |
| 2121 | /* use binary search to find the start */ |
| 2122 | nr = binsearch(NULL, 0, r->ttype, rvals, rvars, rwidth, 0, rci->next - rscan, v, rordering, 0); |
| 2123 | nr = canditer_search(rci, nr + r->hseqbase, true); |
| 2124 | nr = rci->next - nr; |
| 2125 | canditer_setidx(rci, rci->next - nr); |
| 2126 | } else { |
| 2127 | /* scan r for start of range */ |
| 2128 | do { |
| 2129 | canditer_prev(rci); |
| 2130 | nr++; |
| 2131 | } while (rci->next > 0 && |
| 2132 | cmp(v, VALUE(r, canditer_peekprev(rci) - r->hseqbase)) == 0); |
| 2133 | } |
| 2134 | } |
| 2135 | } else { |
| 2136 | lv = canditer_search(&rrci, *(const oid *)v, true); |
| 2137 | lv = canditer_search(rci, lv + r->hseqbase, true); |
| 2138 | nr = (canditer_idx(rci, lv) == *(const oid*)v); |
| 2139 | canditer_setidx(rci, lv); |
| 2140 | } |
| 2141 | /* rci points to first value > v |
| 2142 | * or end of r, and nr is the number of values |
| 2143 | * in r that are equal to v */ |
| 2144 | } |
| 2145 | |
| 2146 | if (nr == 0) { |
| 2147 | /* no entries in r found */ |
| 2148 | if (!(nil_on_miss | only_misses)) { |
| 2149 | if (lscan > 0 && |
| 2150 | (equal_order ? rci->next == rci->ncand : rci->next == 0)) { |
| 2151 | /* nothing more left to match |
| 2152 | * in r */ |
| 2153 | break; |
| 2154 | } |
| 2155 | lskipped = BATcount(r1) > 0; |
| 2156 | canditer_setidx(lci, lci->next + nl); |
| 2157 | continue; |
| 2158 | } |
| 2159 | /* insert a nil to indicate a non-match */ |
| 2160 | insert_nil = true; |
| 2161 | nr = 1; |
| 2162 | if (r2) { |
| 2163 | r2->tnil = true; |
| 2164 | r2->tnonil = false; |
| 2165 | r2->tsorted = false; |
| 2166 | r2->trevsorted = false; |
| 2167 | r2->tseqbase = oid_nil; |
| 2168 | r2->tkey = false; |
| 2169 | } |
| 2170 | } else if (only_misses) { |
| 2171 | /* we had a match, so we're not interested */ |
| 2172 | lskipped = BATcount(r1) > 0; |
| 2173 | canditer_setidx(lci, lci->next + nl); |
| 2174 | continue; |
| 2175 | } else { |
| 2176 | insert_nil = false; |
| 2177 | if (semi) { |
| 2178 | /* for semi-join, only insert single |
| 2179 | * value */ |
| 2180 | nr = 1; |
| 2181 | } |
| 2182 | } |
| 2183 | if (canditer_idx(lci, lci->next + nl - 1) - canditer_idx(lci, lci->next) != nl - 1) { |
| 2184 | /* not all values in the range are |
| 2185 | * candidates */ |
| 2186 | lskipped = true; |
| 2187 | } |
| 2188 | /* make space: nl values in l match nr values in r, so |
| 2189 | * we need to add nl * nr values in the results */ |
| 2190 | MAYBEEXTEND(nl * nr, lci); |
| 2191 | |
| 2192 | /* maintain properties */ |
| 2193 | if (nl > 1) { |
| 2194 | if (r2) { |
| 2195 | /* value occurs multiple times in l, |
| 2196 | * so entry in r will be repeated |
| 2197 | * multiple times: hence r2 is not key |
| 2198 | * and not dense */ |
| 2199 | r2->tkey = false; |
| 2200 | r2->tseqbase = oid_nil; |
| 2201 | } |
| 2202 | /* multiple different values will be inserted |
| 2203 | * in r1 (always in order), so not reverse |
| 2204 | * ordered anymore */ |
| 2205 | r1->trevsorted = false; |
| 2206 | } |
| 2207 | if (nr > 1) { |
| 2208 | /* value occurs multiple times in r, so entry |
| 2209 | * in l will be repeated multiple times: hence |
| 2210 | * r1 is not key and not dense */ |
| 2211 | r1->tkey = false; |
| 2212 | r1->tseqbase = oid_nil; |
| 2213 | if (r2) { |
| 2214 | /* multiple different values will be |
| 2215 | * inserted in r2 (in order), so not |
| 2216 | * reverse ordered anymore */ |
| 2217 | r2->trevsorted = false; |
| 2218 | if (nl > 1) { |
| 2219 | /* multiple values in l match |
| 2220 | * multiple values in r, so an |
| 2221 | * ordered sequence will be |
| 2222 | * inserted multiple times in |
| 2223 | * r2, so r2 is not ordered |
| 2224 | * anymore */ |
| 2225 | r2->tsorted = false; |
| 2226 | } |
| 2227 | } |
| 2228 | } |
| 2229 | if (lscan == 0) { |
| 2230 | /* deduce relative positions of r matches for |
| 2231 | * this and previous value in v */ |
| 2232 | if (prev && r2) { |
| 2233 | /* keyness or r2 can only be assured |
| 2234 | * as long as matched values are |
| 2235 | * ordered */ |
| 2236 | int ord = rordering * cmp(prev, v); |
| 2237 | if (ord < 0) { |
| 2238 | /* previous value in l was |
| 2239 | * less than current */ |
| 2240 | r2->trevsorted = false; |
| 2241 | r2->tkey &= r2->tsorted; |
| 2242 | } else if (ord > 0) { |
| 2243 | /* previous value was |
| 2244 | * greater */ |
| 2245 | r2->tsorted = false; |
| 2246 | r2->tkey &= r2->trevsorted; |
| 2247 | } else { |
| 2248 | /* value can be equal if |
| 2249 | * intervening values in l |
| 2250 | * didn't match anything; if |
| 2251 | * multiple values match in r, |
| 2252 | * r2 won't be sorted */ |
| 2253 | r2->tkey = false; |
| 2254 | if (nr > 1) { |
| 2255 | r2->tsorted = false; |
| 2256 | r2->trevsorted = false; |
| 2257 | } |
| 2258 | } |
| 2259 | } |
| 2260 | prev = v; |
| 2261 | } |
| 2262 | if (BATcount(r1) > 0) { |
| 2263 | /* a new, higher value will be inserted into |
| 2264 | * r1, so r1 is not reverse ordered anymore */ |
| 2265 | r1->trevsorted = false; |
| 2266 | if (r2) { |
| 2267 | /* depending on whether l and r are |
| 2268 | * ordered the same or not, a new |
| 2269 | * higher or lower value will be added |
| 2270 | * to r2 */ |
| 2271 | if (equal_order) |
| 2272 | r2->trevsorted = false; |
| 2273 | else { |
| 2274 | r2->tsorted = false; |
| 2275 | r2->tseqbase = oid_nil; |
| 2276 | } |
| 2277 | } |
| 2278 | /* if there is a left candidate list, it may |
| 2279 | * be that the next value added isn't |
| 2280 | * consecutive with the last one */ |
| 2281 | if (lskipped || |
| 2282 | ((oid *) r1->theap.base)[r1->batCount - 1] + 1 != canditer_peek(lci)) |
| 2283 | r1->tseqbase = oid_nil; |
| 2284 | } |
| 2285 | |
| 2286 | /* insert values: first the left output */ |
| 2287 | for (i = 0; i < nl; i++) { |
| 2288 | lv = canditer_next(lci); |
| 2289 | for (j = 0; j < nr; j++) |
| 2290 | APPEND(r1, lv); |
| 2291 | } |
| 2292 | /* then the right output, various different ways of |
| 2293 | * doing it */ |
| 2294 | if (r2 == NULL) { |
| 2295 | /* nothing to do */ |
| 2296 | } else if (insert_nil) { |
| 2297 | do { |
| 2298 | for (i = 0; i < nr; i++) { |
| 2299 | APPEND(r2, oid_nil); |
| 2300 | } |
| 2301 | } while (--nl > 0); |
| 2302 | } else if (equal_order) { |
| 2303 | struct canditer ci = *rci; /* work on copy */ |
| 2304 | if (r2->batCount > 0 && |
| 2305 | BATtdense(r2) && |
| 2306 | ((oid *) r2->theap.base)[r2->batCount - 1] + 1 != canditer_idx(&ci, ci.next - nr)) |
| 2307 | r2->tseqbase = oid_nil; |
| 2308 | do { |
| 2309 | canditer_setidx(&ci, ci.next - nr); |
| 2310 | for (i = 0; i < nr; i++) { |
| 2311 | APPEND(r2, canditer_next(&ci)); |
| 2312 | } |
| 2313 | } while (--nl > 0); |
| 2314 | } else { |
| 2315 | if (r2->batCount > 0 && |
| 2316 | BATtdense(r2) && |
| 2317 | ((oid *) r2->theap.base)[r2->batCount - 1] + 1 != canditer_peek(rci)) |
| 2318 | r2->tseqbase = oid_nil; |
| 2319 | do { |
| 2320 | struct canditer ci = *rci; /* work on copy */ |
| 2321 | for (i = 0; i < nr; i++) { |
| 2322 | APPEND(r2, canditer_next(&ci)); |
| 2323 | } |
| 2324 | } while (--nl > 0); |
| 2325 | } |
| 2326 | } |
| 2327 | /* also set other bits of heap to correct value to indicate size */ |
| 2328 | BATsetcount(r1, BATcount(r1)); |
| 2329 | r1->tseqbase = oid_nil; |
| 2330 | if (r2) { |
| 2331 | BATsetcount(r2, BATcount(r2)); |
| 2332 | assert(BATcount(r1) == BATcount(r2)); |
| 2333 | r2->tseqbase = oid_nil; |
| 2334 | } |
| 2335 | if (BATcount(r1) > 0) { |
| 2336 | if (BATtdense(r1)) |
| 2337 | r1->tseqbase = ((oid *) r1->theap.base)[0]; |
| 2338 | if (r2 && BATtdense(r2)) |
| 2339 | r2->tseqbase = ((oid *) r2->theap.base)[0]; |
| 2340 | } else { |
| 2341 | r1->tseqbase = 0; |
| 2342 | if (r2) { |
| 2343 | r2->tseqbase = 0; |
| 2344 | } |
| 2345 | } |
| 2346 | ALGODEBUG fprintf(stderr, "#%s: %s(l="ALGOBATFMT "," |
| 2347 | "r="ALGOBATFMT ",sl="ALGOOPTBATFMT "," |
| 2348 | "sr="ALGOOPTBATFMT ",nil_matches=%d," |
| 2349 | "nil_on_miss=%d,semi=%d,only_misses=%d," |
| 2350 | "not_in=%d)%s %s " |
| 2351 | "-> ("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 2352 | MT_thread_getname(), __func__, |
| 2353 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 2354 | ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr), |
| 2355 | nil_matches, nil_on_miss, semi, only_misses, not_in, |
| 2356 | swapped ? " swapped": "", reason, |
| 2357 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 2358 | GDKusec() - t0); |
| 2359 | |
| 2360 | return GDK_SUCCEED; |
| 2361 | |
| 2362 | bailout: |
| 2363 | BBPreclaim(r1); |
| 2364 | BBPreclaim(r2); |
| 2365 | return GDK_FAIL; |
| 2366 | } |
| 2367 | |
| 2368 | #define HASHLOOPBODY() \ |
| 2369 | do { \ |
| 2370 | MAYBEEXTEND(1, lci); \ |
| 2371 | APPEND(r1, lo); \ |
| 2372 | if (r2) \ |
| 2373 | APPEND(r2, ro); \ |
| 2374 | nr++; \ |
| 2375 | } while (false) |
| 2376 | |
| 2377 | #define HASHloop_bound_TYPE(vals, h, hb, v, lo, hi, TYPE) \ |
| 2378 | for (hb = HASHget(h, hash_##TYPE(h, &v)); \ |
| 2379 | hb != HASHnil(h); \ |
| 2380 | hb = HASHgetlink(h,hb)) \ |
| 2381 | if (hb >= (lo) && hb < (hi) && \ |
| 2382 | v == vals[hb]) |
| 2383 | |
| 2384 | #define HASHloop_bound(bi, h, hb, v, lo, hi) \ |
| 2385 | for (hb = HASHget(h, HASHprobe((h), v)); \ |
| 2386 | hb != HASHnil(h); \ |
| 2387 | hb = HASHgetlink(h,hb)) \ |
| 2388 | if (hb >= (lo) && hb < (hi) && \ |
| 2389 | (cmp == NULL || \ |
| 2390 | (*cmp)(v, BUNtail(bi, hb)) == 0)) |
| 2391 | |
| 2392 | #define HASHJOIN(TYPE) \ |
| 2393 | do { \ |
| 2394 | TYPE *rvals = Tloc(r, 0); \ |
| 2395 | TYPE *lvals = Tloc(l, 0); \ |
| 2396 | TYPE v; \ |
| 2397 | while (lci->next < lci->ncand) { \ |
| 2398 | lo = canditer_next(lci); \ |
| 2399 | v = lvals[lo - l->hseqbase]; \ |
| 2400 | nr = 0; \ |
| 2401 | if ((!nil_matches || not_in) && is_##TYPE##_nil(v)) { \ |
| 2402 | /* no match */ \ |
| 2403 | if (not_in) \ |
| 2404 | continue; \ |
| 2405 | } else if (sr) { \ |
| 2406 | for (rb = HASHget(hsh, hash_##TYPE(hsh, &v)); \ |
| 2407 | rb != HASHnil(hsh); \ |
| 2408 | rb = HASHgetlink(hsh, rb)) { \ |
| 2409 | ro = BUNtoid(sr, rb); \ |
| 2410 | if (v != rvals[ro - r->hseqbase]) \ |
| 2411 | continue; \ |
| 2412 | if (only_misses) { \ |
| 2413 | nr++; \ |
| 2414 | break; \ |
| 2415 | } \ |
| 2416 | HASHLOOPBODY(); \ |
| 2417 | if (semi) \ |
| 2418 | break; \ |
| 2419 | } \ |
| 2420 | } else { \ |
| 2421 | HASHloop_bound_TYPE(rvals, hsh, rb, v, rl, rh, TYPE) { \ |
| 2422 | ro = (oid) (rb - rl + rseq); \ |
| 2423 | if (only_misses) { \ |
| 2424 | nr++; \ |
| 2425 | break; \ |
| 2426 | } \ |
| 2427 | HASHLOOPBODY(); \ |
| 2428 | if (semi) \ |
| 2429 | break; \ |
| 2430 | } \ |
| 2431 | } \ |
| 2432 | if (nr == 0) { \ |
| 2433 | if (only_misses) { \ |
| 2434 | nr = 1; \ |
| 2435 | MAYBEEXTEND(1, lci); \ |
| 2436 | APPEND(r1, lo); \ |
| 2437 | if (lskipped) \ |
| 2438 | r1->tseqbase = oid_nil; \ |
| 2439 | } else if (nil_on_miss) { \ |
| 2440 | nr = 1; \ |
| 2441 | r2->tnil = true; \ |
| 2442 | r2->tnonil = false; \ |
| 2443 | r2->tkey = false; \ |
| 2444 | MAYBEEXTEND(1, lci); \ |
| 2445 | APPEND(r1, lo); \ |
| 2446 | APPEND(r2, oid_nil); \ |
| 2447 | } else { \ |
| 2448 | lskipped = BATcount(r1) > 0; \ |
| 2449 | } \ |
| 2450 | } else if (only_misses) { \ |
| 2451 | lskipped = BATcount(r1) > 0; \ |
| 2452 | } else { \ |
| 2453 | if (lskipped) { \ |
| 2454 | /* note, we only get here in an \ |
| 2455 | * iteration *after* lskipped was \ |
| 2456 | * first set to true, i.e. we did \ |
| 2457 | * indeed skip values in l */ \ |
| 2458 | r1->tseqbase = oid_nil; \ |
| 2459 | } \ |
| 2460 | if (nr > 1) { \ |
| 2461 | r1->tkey = false; \ |
| 2462 | r1->tseqbase = oid_nil; \ |
| 2463 | } \ |
| 2464 | } \ |
| 2465 | if (nr > 0 && BATcount(r1) > nr) \ |
| 2466 | r1->trevsorted = false; \ |
| 2467 | } \ |
| 2468 | } while (0) |
| 2469 | |
| 2470 | /* Implementation of join using a hash lookup of values in the right |
| 2471 | * column. */ |
| 2472 | static gdk_return |
| 2473 | hashjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, |
| 2474 | struct canditer *restrict lci, struct canditer *restrict rci, |
| 2475 | bool nil_matches, bool nil_on_miss, bool semi, bool only_misses, |
| 2476 | bool not_in, |
| 2477 | BUN estimate, lng t0, bool swapped, bool phash, const char *reason) |
| 2478 | { |
| 2479 | oid lo, ro; |
| 2480 | BATiter ri; |
| 2481 | BUN rb; |
| 2482 | BUN rl, rh; |
| 2483 | oid rseq; |
| 2484 | BUN nr; |
| 2485 | const char *lvals; |
| 2486 | const char *lvars; |
| 2487 | int lwidth; |
| 2488 | const void *nil = ATOMnilptr(l->ttype); |
| 2489 | int (*cmp)(const void *, const void *) = ATOMcompare(l->ttype); |
| 2490 | oid lval = oid_nil; /* hold value if l is dense */ |
| 2491 | const char *v = (const char *) &lval; |
| 2492 | bool lskipped = false; /* whether we skipped values in l */ |
| 2493 | Hash *restrict hsh; |
| 2494 | |
| 2495 | assert(!BATtvoid(r)); |
| 2496 | assert(ATOMtype(l->ttype) == ATOMtype(r->ttype)); |
| 2497 | assert(sl == NULL || sl->tsorted); |
| 2498 | assert(sr == NULL || sr->tsorted); |
| 2499 | |
| 2500 | int t = ATOMbasetype(r->ttype); |
| 2501 | if (r->ttype == TYPE_void || l->ttype == TYPE_void) |
| 2502 | t = TYPE_void; |
| 2503 | |
| 2504 | lwidth = l->twidth; |
| 2505 | lvals = (const char *) Tloc(l, 0); |
| 2506 | if (l->tvarsized && l->ttype) { |
| 2507 | assert(r->tvarsized && r->ttype); |
| 2508 | lvars = l->tvheap->base; |
| 2509 | } else { |
| 2510 | assert(!r->tvarsized || !r->ttype); |
| 2511 | lvars = NULL; |
| 2512 | } |
| 2513 | /* offset to convert BUN to OID for value in right column */ |
| 2514 | rseq = r->hseqbase; |
| 2515 | |
| 2516 | if (lci->ncand == 0 || rci->ncand== 0) |
| 2517 | return nomatch(r1p, r2p, l, r, lci, |
| 2518 | nil_on_miss, only_misses, "hashjoin", t0); |
| 2519 | |
| 2520 | BUN maxsize = joininitresults(r1p, r2p, lci->ncand, rci->ncand, |
| 2521 | l->tkey, r->tkey, semi, nil_on_miss, |
| 2522 | only_misses, estimate); |
| 2523 | if (maxsize == BUN_NONE) |
| 2524 | return GDK_FAIL; |
| 2525 | |
| 2526 | BAT *r1 = *r1p; |
| 2527 | BAT *r2 = r2p ? *r2p : NULL; |
| 2528 | |
| 2529 | rl = rci->seq - r->hseqbase; |
| 2530 | rh = canditer_last(rci) + 1 - r->hseqbase; |
| 2531 | if (phash) { |
| 2532 | BAT *b = BBPdescriptor(VIEWtparent(r)); |
| 2533 | assert(sr == NULL); |
| 2534 | ALGODEBUG fprintf(stderr, "#%s: %s(%s): using " |
| 2535 | "parent("ALGOBATFMT ") for hash\n", |
| 2536 | MT_thread_getname(), __func__, |
| 2537 | BATgetId(r), ALGOBATPAR(b)); |
| 2538 | rl += (BUN) ((r->theap.base - b->theap.base) >> r->tshift); |
| 2539 | rh += rl; |
| 2540 | r = b; |
| 2541 | } |
| 2542 | |
| 2543 | if (sr) { |
| 2544 | if (BATtdense(sr) && |
| 2545 | BATcheckhash(r) && |
| 2546 | BATcount(r) / ((size_t *) r->thash->heap.base)[5] * lci->ncand < lci->ncand + rci->ncand) { |
| 2547 | ALGODEBUG fprintf(stderr, "#%s: %s(%s): using " |
| 2548 | "existing hash with candidate list\n", |
| 2549 | MT_thread_getname(), __func__, |
| 2550 | BATgetId(r)); |
| 2551 | hsh = r->thash; |
| 2552 | sr = NULL; |
| 2553 | } else { |
| 2554 | int len; |
| 2555 | char ext[32]; |
| 2556 | assert(!phash); |
| 2557 | ALGODEBUG fprintf(stderr, "#%s: %s(%s): creating " |
| 2558 | "hash for candidate list\n", |
| 2559 | MT_thread_getname(), __func__, |
| 2560 | BATgetId(r)); |
| 2561 | len = snprintf(ext, sizeof(ext), "thash%x", sr->batCacheid); |
| 2562 | if (len == -1 || len >= (int) sizeof(ext)) |
| 2563 | goto bailout; |
| 2564 | if ((hsh = BAThash_impl(r, sr, ext)) == NULL) { |
| 2565 | goto bailout; |
| 2566 | } |
| 2567 | } |
| 2568 | } else { |
| 2569 | if (BAThash(r) != GDK_SUCCEED) { |
| 2570 | hsh = NULL; |
| 2571 | goto bailout; |
| 2572 | } |
| 2573 | hsh = r->thash; |
| 2574 | } |
| 2575 | ri = bat_iterator(r); |
| 2576 | |
| 2577 | if (not_in && !r->tnonil) { |
| 2578 | /* check whether there is a nil on the right, since if |
| 2579 | * so, we should return an empty result */ |
| 2580 | if (sr) { |
| 2581 | for (rb = HASHget(hsh, HASHprobe(hsh, nil)); |
| 2582 | rb != HASHnil(hsh); |
| 2583 | rb = HASHgetlink(hsh, rb)) { |
| 2584 | ro = BUNtoid(sr, rb); |
| 2585 | if ((*cmp)(v, BUNtail(ri, ro - r->hseqbase)) != 0) { |
| 2586 | HEAPfree(&hsh->heap, true); |
| 2587 | GDKfree(hsh); |
| 2588 | return nomatch(r1p, r2p, l, r, lci, |
| 2589 | false, false, "hashjoin", t0); |
| 2590 | } |
| 2591 | } |
| 2592 | } else { |
| 2593 | HASHloop_bound(ri, hsh, rb, nil, rl, rh) { |
| 2594 | return nomatch(r1p, r2p, l, r, lci, |
| 2595 | false, false, "hashjoin", t0); |
| 2596 | } |
| 2597 | } |
| 2598 | } |
| 2599 | |
| 2600 | /* basic properties will be adjusted if necessary later on, |
| 2601 | * they were initially set by joininitresults() */ |
| 2602 | |
| 2603 | if (r2) { |
| 2604 | r2->tkey = l->tkey; |
| 2605 | /* r2 is not likely to be sorted (although it is |
| 2606 | * certainly possible) */ |
| 2607 | r2->tsorted = false; |
| 2608 | r2->trevsorted = false; |
| 2609 | r2->tseqbase = oid_nil; |
| 2610 | } |
| 2611 | |
| 2612 | if (sl && !BATtdense(sl)) |
| 2613 | r1->tseqbase = oid_nil; |
| 2614 | |
| 2615 | switch (t) { |
| 2616 | case TYPE_int: |
| 2617 | HASHJOIN(int); |
| 2618 | break; |
| 2619 | case TYPE_lng: |
| 2620 | HASHJOIN(lng); |
| 2621 | break; |
| 2622 | default: |
| 2623 | while (lci->next < lci->ncand) { |
| 2624 | lo = canditer_next(lci); |
| 2625 | if (BATtvoid(l)) { |
| 2626 | if (BATtdense(l)) |
| 2627 | lval = lo - l->hseqbase + l->tseqbase; |
| 2628 | } else { |
| 2629 | v = VALUE(l, lo - l->hseqbase); |
| 2630 | } |
| 2631 | nr = 0; |
| 2632 | if ((!nil_matches || not_in) && cmp(v, nil) == 0) { |
| 2633 | /* no match */ |
| 2634 | if (not_in) |
| 2635 | continue; |
| 2636 | } else if (sr) { |
| 2637 | for (rb = HASHget(hsh, HASHprobe(hsh, v)); |
| 2638 | rb != HASHnil(hsh); |
| 2639 | rb = HASHgetlink(hsh, rb)) { |
| 2640 | ro = BUNtoid(sr, rb); |
| 2641 | if ((*cmp)(v, BUNtail(ri, ro - r->hseqbase)) != 0) |
| 2642 | continue; |
| 2643 | if (only_misses) { |
| 2644 | nr++; |
| 2645 | break; |
| 2646 | } |
| 2647 | HASHLOOPBODY(); |
| 2648 | if (semi) |
| 2649 | break; |
| 2650 | } |
| 2651 | } else { |
| 2652 | HASHloop_bound(ri, hsh, rb, v, rl, rh) { |
| 2653 | ro = (oid) (rb - rl + rseq); |
| 2654 | if (only_misses) { |
| 2655 | nr++; |
| 2656 | break; |
| 2657 | } |
| 2658 | HASHLOOPBODY(); |
| 2659 | if (semi) |
| 2660 | break; |
| 2661 | } |
| 2662 | } |
| 2663 | if (nr == 0) { |
| 2664 | if (only_misses) { |
| 2665 | nr = 1; |
| 2666 | MAYBEEXTEND(1, lci); |
| 2667 | APPEND(r1, lo); |
| 2668 | if (lskipped) |
| 2669 | r1->tseqbase = oid_nil; |
| 2670 | } else if (nil_on_miss) { |
| 2671 | nr = 1; |
| 2672 | r2->tnil = true; |
| 2673 | r2->tnonil = false; |
| 2674 | r2->tkey = false; |
| 2675 | MAYBEEXTEND(1, lci); |
| 2676 | APPEND(r1, lo); |
| 2677 | APPEND(r2, oid_nil); |
| 2678 | } else { |
| 2679 | lskipped = BATcount(r1) > 0; |
| 2680 | } |
| 2681 | } else if (only_misses) { |
| 2682 | lskipped = BATcount(r1) > 0; |
| 2683 | } else { |
| 2684 | if (lskipped) { |
| 2685 | /* note, we only get here in an |
| 2686 | * iteration *after* lskipped was |
| 2687 | * first set to true, i.e. we did |
| 2688 | * indeed skip values in l */ |
| 2689 | r1->tseqbase = oid_nil; |
| 2690 | } |
| 2691 | if (nr > 1) { |
| 2692 | r1->tkey = false; |
| 2693 | r1->tseqbase = oid_nil; |
| 2694 | } |
| 2695 | } |
| 2696 | if (nr > 0 && BATcount(r1) > nr) |
| 2697 | r1->trevsorted = false; |
| 2698 | } |
| 2699 | break; |
| 2700 | } |
| 2701 | if (sr) { |
| 2702 | HEAPfree(&hsh->heap, true); |
| 2703 | GDKfree(hsh); |
| 2704 | } |
| 2705 | /* also set other bits of heap to correct value to indicate size */ |
| 2706 | BATsetcount(r1, BATcount(r1)); |
| 2707 | if (BATcount(r1) <= 1) { |
| 2708 | r1->tsorted = true; |
| 2709 | r1->trevsorted = true; |
| 2710 | r1->tkey = true; |
| 2711 | r1->tseqbase = 0; |
| 2712 | } |
| 2713 | if (r2) { |
| 2714 | BATsetcount(r2, BATcount(r2)); |
| 2715 | assert(BATcount(r1) == BATcount(r2)); |
| 2716 | if (BATcount(r2) <= 1) { |
| 2717 | r2->tsorted = true; |
| 2718 | r2->trevsorted = true; |
| 2719 | r2->tkey = true; |
| 2720 | r2->tseqbase = 0; |
| 2721 | } |
| 2722 | } |
| 2723 | if (BATcount(r1) > 0) { |
| 2724 | if (BATtdense(r1)) |
| 2725 | r1->tseqbase = ((oid *) r1->theap.base)[0]; |
| 2726 | if (r2 && BATtdense(r2)) |
| 2727 | r2->tseqbase = ((oid *) r2->theap.base)[0]; |
| 2728 | } else { |
| 2729 | r1->tseqbase = 0; |
| 2730 | if (r2) { |
| 2731 | r2->tseqbase = 0; |
| 2732 | } |
| 2733 | } |
| 2734 | ALGODEBUG fprintf(stderr, "#%s: %s(l="ALGOBATFMT "," |
| 2735 | "r="ALGOBATFMT ",sl="ALGOOPTBATFMT "," |
| 2736 | "sr="ALGOOPTBATFMT ",nil_matches=%d," |
| 2737 | "nil_on_miss=%d,semi=%d,only_misses=%d," |
| 2738 | "not_in=%d)%s %s " |
| 2739 | "-> ("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 2740 | MT_thread_getname(), __func__, |
| 2741 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 2742 | ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr), |
| 2743 | nil_matches, nil_on_miss, semi, only_misses, not_in, |
| 2744 | swapped ? " swapped": "", reason, |
| 2745 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 2746 | GDKusec() - t0); |
| 2747 | |
| 2748 | return GDK_SUCCEED; |
| 2749 | |
| 2750 | bailout: |
| 2751 | if (sr && hsh) { |
| 2752 | HEAPfree(&hsh->heap, true); |
| 2753 | GDKfree(hsh); |
| 2754 | } |
| 2755 | BBPreclaim(r1); |
| 2756 | BBPreclaim(r2); |
| 2757 | return GDK_FAIL; |
| 2758 | } |
| 2759 | |
| 2760 | #define MASK_EQ 1 |
| 2761 | #define MASK_LT 2 |
| 2762 | #define MASK_GT 4 |
| 2763 | #define MASK_LE (MASK_EQ | MASK_LT) |
| 2764 | #define MASK_GE (MASK_EQ | MASK_GT) |
| 2765 | #define MASK_NE (MASK_LT | MASK_GT) |
| 2766 | |
| 2767 | static gdk_return |
| 2768 | thetajoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, int opcode, BUN estimate, lng t0) |
| 2769 | { |
| 2770 | struct canditer lci, rci; |
| 2771 | BUN lcnt, rcnt; |
| 2772 | const char *lvals, *rvals; |
| 2773 | const char *lvars, *rvars; |
| 2774 | int lwidth, rwidth; |
| 2775 | const void *nil = ATOMnilptr(l->ttype); |
| 2776 | int (*cmp)(const void *, const void *) = ATOMcompare(l->ttype); |
| 2777 | const void *vl, *vr; |
| 2778 | oid lastr = 0; /* last value inserted into r2 */ |
| 2779 | BUN nr; |
| 2780 | oid lo, ro; |
| 2781 | int c; |
| 2782 | bool lskipped = false; /* whether we skipped values in l */ |
| 2783 | lng loff = 0, roff = 0; |
| 2784 | oid lval = oid_nil, rval = oid_nil; |
| 2785 | |
| 2786 | ALGODEBUG fprintf(stderr, "#thetajoin(l="ALGOBATFMT "," |
| 2787 | "r="ALGOBATFMT ",sl="ALGOOPTBATFMT "," |
| 2788 | "sr="ALGOOPTBATFMT ",op=%s%s%s)\n", |
| 2789 | ALGOBATPAR(l), ALGOBATPAR(r), ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr), |
| 2790 | opcode & MASK_LT ? "<": "", |
| 2791 | opcode & MASK_GT ? ">": "", |
| 2792 | opcode & MASK_EQ ? "=": ""); |
| 2793 | |
| 2794 | assert(ATOMtype(l->ttype) == ATOMtype(r->ttype)); |
| 2795 | assert(sl == NULL || sl->tsorted); |
| 2796 | assert(sr == NULL || sr->tsorted); |
| 2797 | assert((opcode & (MASK_EQ | MASK_LT | MASK_GT)) != 0); |
| 2798 | |
| 2799 | lcnt = canditer_init(&lci, l, sl); |
| 2800 | rcnt = canditer_init(&rci, r, sr); |
| 2801 | |
| 2802 | lvals = BATtvoid(l) ? NULL : (const char *) Tloc(l, 0); |
| 2803 | rvals = BATtvoid(r) ? NULL : (const char *) Tloc(r, 0); |
| 2804 | if (l->tvarsized && l->ttype) { |
| 2805 | assert(r->tvarsized && r->ttype); |
| 2806 | lvars = l->tvheap->base; |
| 2807 | rvars = r->tvheap->base; |
| 2808 | } else { |
| 2809 | assert(!r->tvarsized || !r->ttype); |
| 2810 | lvars = rvars = NULL; |
| 2811 | } |
| 2812 | lwidth = l->twidth; |
| 2813 | rwidth = r->twidth; |
| 2814 | |
| 2815 | if (BATtvoid(l)) { |
| 2816 | if (!BATtdense(l)) { |
| 2817 | /* trivial: nils don't match anything */ |
| 2818 | return nomatch(r1p, r2p, l, r, &lci, |
| 2819 | false, false, "thetajoin", t0); |
| 2820 | } |
| 2821 | loff = (lng) l->tseqbase - (lng) l->hseqbase; |
| 2822 | } |
| 2823 | if (BATtvoid(r)) { |
| 2824 | if (!BATtdense(r)) { |
| 2825 | /* trivial: nils don't match anything */ |
| 2826 | return nomatch(r1p, r2p, l, r, &lci, |
| 2827 | false, false, "thetajoin", t0); |
| 2828 | } |
| 2829 | roff = (lng) r->tseqbase - (lng) r->hseqbase; |
| 2830 | } |
| 2831 | |
| 2832 | BUN maxsize = joininitresults(r1p, r2p, lcnt, rcnt, false, false, |
| 2833 | false, false, false, estimate); |
| 2834 | if (maxsize == BUN_NONE) |
| 2835 | return GDK_FAIL; |
| 2836 | BAT *r1 = *r1p; |
| 2837 | BAT *r2 = r2p ? *r2p : NULL; |
| 2838 | |
| 2839 | r1->tkey = true; |
| 2840 | r1->tsorted = true; |
| 2841 | r1->trevsorted = true; |
| 2842 | if (r2) { |
| 2843 | r2->tkey = true; |
| 2844 | r2->tsorted = true; |
| 2845 | r2->trevsorted = true; |
| 2846 | } |
| 2847 | |
| 2848 | /* nested loop implementation for theta join */ |
| 2849 | vl = &lval; |
| 2850 | vr = &rval; |
| 2851 | for (BUN li = 0; li < lci.ncand; li++) { |
| 2852 | lo = canditer_next(&lci); |
| 2853 | if (lvals) |
| 2854 | vl = VALUE(l, lo - l->hseqbase); |
| 2855 | else |
| 2856 | lval = (oid) ((lng) lo + loff); |
| 2857 | nr = 0; |
| 2858 | if (cmp(vl, nil) != 0) { |
| 2859 | canditer_reset(&rci); |
| 2860 | for (BUN ri = 0; ri < rci.ncand; ri++) { |
| 2861 | ro = canditer_next(&rci); |
| 2862 | if (rvals) |
| 2863 | vr = VALUE(r, ro - r->hseqbase); |
| 2864 | else |
| 2865 | rval = (oid) ((lng) ro + roff); |
| 2866 | if (cmp(vr, nil) == 0) |
| 2867 | continue; |
| 2868 | c = cmp(vl, vr); |
| 2869 | if (!((opcode & MASK_LT && c < 0) || |
| 2870 | (opcode & MASK_GT && c > 0) || |
| 2871 | (opcode & MASK_EQ && c == 0))) |
| 2872 | continue; |
| 2873 | MAYBEEXTEND(1, &lci); |
| 2874 | if (BATcount(r1) > 0) { |
| 2875 | if (r2 && lastr + 1 != ro) |
| 2876 | r2->tseqbase = oid_nil; |
| 2877 | if (nr == 0) { |
| 2878 | r1->trevsorted = false; |
| 2879 | if (r2 == NULL) { |
| 2880 | /* nothing */ |
| 2881 | } else if (lastr > ro) { |
| 2882 | r2->tsorted = false; |
| 2883 | r2->tkey = false; |
| 2884 | } else if (lastr < ro) { |
| 2885 | r2->trevsorted = false; |
| 2886 | } else { |
| 2887 | r2->tkey = false; |
| 2888 | } |
| 2889 | } |
| 2890 | } |
| 2891 | APPEND(r1, lo); |
| 2892 | if (r2) { |
| 2893 | APPEND(r2, ro); |
| 2894 | } |
| 2895 | lastr = ro; |
| 2896 | nr++; |
| 2897 | } |
| 2898 | } |
| 2899 | if (nr > 1) { |
| 2900 | r1->tkey = false; |
| 2901 | r1->tseqbase = oid_nil; |
| 2902 | if (r2) { |
| 2903 | r2->trevsorted = false; |
| 2904 | } |
| 2905 | } else if (nr == 0) { |
| 2906 | lskipped = BATcount(r1) > 0; |
| 2907 | } else if (lskipped) { |
| 2908 | r1->tseqbase = oid_nil; |
| 2909 | } |
| 2910 | } |
| 2911 | /* also set other bits of heap to correct value to indicate size */ |
| 2912 | BATsetcount(r1, BATcount(r1)); |
| 2913 | if (r2) { |
| 2914 | BATsetcount(r2, BATcount(r2)); |
| 2915 | assert(BATcount(r1) == BATcount(r2)); |
| 2916 | } |
| 2917 | if (BATcount(r1) > 0) { |
| 2918 | if (BATtdense(r1)) |
| 2919 | r1->tseqbase = ((oid *) r1->theap.base)[0]; |
| 2920 | if (r2 && BATtdense(r2)) |
| 2921 | r2->tseqbase = ((oid *) r2->theap.base)[0]; |
| 2922 | } else { |
| 2923 | r1->tseqbase = 0; |
| 2924 | if (r2) { |
| 2925 | r2->tseqbase = 0; |
| 2926 | } |
| 2927 | } |
| 2928 | ALGODEBUG fprintf(stderr, "#thetajoin(l=%s,r=%s)=("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 2929 | BATgetId(l), BATgetId(r), |
| 2930 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 2931 | GDKusec() - t0); |
| 2932 | return GDK_SUCCEED; |
| 2933 | |
| 2934 | bailout: |
| 2935 | BBPreclaim(r1); |
| 2936 | BBPreclaim(r2); |
| 2937 | return GDK_FAIL; |
| 2938 | } |
| 2939 | |
| 2940 | static gdk_return |
| 2941 | bandjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, |
| 2942 | const void *c1, const void *c2, bool li, bool hi, BUN estimate, |
| 2943 | lng t0) |
| 2944 | { |
| 2945 | BUN lcnt, rcnt; |
| 2946 | struct canditer lci, rci; |
| 2947 | const char *lvals, *rvals; |
| 2948 | int lwidth, rwidth; |
| 2949 | int t; |
| 2950 | const void *nil = ATOMnilptr(l->ttype); |
| 2951 | int (*cmp)(const void *, const void *) = ATOMcompare(l->ttype); |
| 2952 | const char *vl, *vr; |
| 2953 | oid lastr = 0; /* last value inserted into r2 */ |
| 2954 | BUN nr; |
| 2955 | oid lo, ro; |
| 2956 | bool lskipped = false; /* whether we skipped values in l */ |
| 2957 | BUN nils = 0; /* needed for XXX_WITH_CHECK macros */ |
| 2958 | |
| 2959 | assert(ATOMtype(l->ttype) == ATOMtype(r->ttype)); |
| 2960 | assert(sl == NULL || sl->tsorted); |
| 2961 | assert(sr == NULL || sr->tsorted); |
| 2962 | |
| 2963 | t = ATOMtype(l->ttype); |
| 2964 | t = ATOMbasetype(t); |
| 2965 | |
| 2966 | lcnt = canditer_init(&lci, l, sl); |
| 2967 | rcnt = canditer_init(&rci, r, sr); |
| 2968 | |
| 2969 | if (lcnt == 0 || rcnt == 0) |
| 2970 | return nomatch(r1p, r2p, l, r, &lci, |
| 2971 | false, false, "bandjoin", t0); |
| 2972 | |
| 2973 | switch (t) { |
| 2974 | case TYPE_bte: |
| 2975 | if (is_bte_nil(*(const bte *)c1) || |
| 2976 | is_bte_nil(*(const bte *)c2) || |
| 2977 | -*(const bte *)c1 > *(const bte *)c2 || |
| 2978 | ((!hi || !li) && -*(const bte *)c1 == *(const bte *)c2)) |
| 2979 | return nomatch(r1p, r2p, l, r, &lci, |
| 2980 | false, false, "bandjoin", t0); |
| 2981 | break; |
| 2982 | case TYPE_sht: |
| 2983 | if (is_sht_nil(*(const sht *)c1) || |
| 2984 | is_sht_nil(*(const sht *)c2) || |
| 2985 | -*(const sht *)c1 > *(const sht *)c2 || |
| 2986 | ((!hi || !li) && -*(const sht *)c1 == *(const sht *)c2)) |
| 2987 | return nomatch(r1p, r2p, l, r, &lci, |
| 2988 | false, false, "bandjoin", t0); |
| 2989 | break; |
| 2990 | case TYPE_int: |
| 2991 | if (is_int_nil(*(const int *)c1) || |
| 2992 | is_int_nil(*(const int *)c2) || |
| 2993 | -*(const int *)c1 > *(const int *)c2 || |
| 2994 | ((!hi || !li) && -*(const int *)c1 == *(const int *)c2)) |
| 2995 | return nomatch(r1p, r2p, l, r, &lci, |
| 2996 | false, false, "bandjoin", t0); |
| 2997 | break; |
| 2998 | case TYPE_lng: |
| 2999 | if (is_lng_nil(*(const lng *)c1) || |
| 3000 | is_lng_nil(*(const lng *)c2) || |
| 3001 | -*(const lng *)c1 > *(const lng *)c2 || |
| 3002 | ((!hi || !li) && -*(const lng *)c1 == *(const lng *)c2)) |
| 3003 | return nomatch(r1p, r2p, l, r, &lci, |
| 3004 | false, false, "bandjoin", t0); |
| 3005 | break; |
| 3006 | #ifdef HAVE_HGE |
| 3007 | case TYPE_hge: |
| 3008 | if (is_hge_nil(*(const hge *)c1) || |
| 3009 | is_hge_nil(*(const hge *)c2) || |
| 3010 | -*(const hge *)c1 > *(const hge *)c2 || |
| 3011 | ((!hi || !li) && -*(const hge *)c1 == *(const hge *)c2)) |
| 3012 | return nomatch(r1p, r2p, l, r, &lci, |
| 3013 | false, false, "bandjoin", t0); |
| 3014 | break; |
| 3015 | #endif |
| 3016 | case TYPE_flt: |
| 3017 | if (is_flt_nil(*(const flt *)c1) || |
| 3018 | is_flt_nil(*(const flt *)c2) || |
| 3019 | -*(const flt *)c1 > *(const flt *)c2 || |
| 3020 | ((!hi || !li) && -*(const flt *)c1 == *(const flt *)c2)) |
| 3021 | return nomatch(r1p, r2p, l, r, &lci, |
| 3022 | false, false, "bandjoin", t0); |
| 3023 | break; |
| 3024 | case TYPE_dbl: |
| 3025 | if (is_dbl_nil(*(const dbl *)c1) || |
| 3026 | is_dbl_nil(*(const dbl *)c2) || |
| 3027 | -*(const dbl *)c1 > *(const dbl *)c2 || |
| 3028 | ((!hi || !li) && -*(const dbl *)c1 == *(const dbl *)c2)) |
| 3029 | return nomatch(r1p, r2p, l, r, &lci, |
| 3030 | false, false, "bandjoin", t0); |
| 3031 | break; |
| 3032 | default: |
| 3033 | GDKerror("BATbandjoin: unsupported type\n"); |
| 3034 | return GDK_FAIL; |
| 3035 | } |
| 3036 | |
| 3037 | BUN maxsize = joininitresults(r1p, r2p, lcnt, rcnt, false, false, |
| 3038 | false, false, false, estimate); |
| 3039 | if (maxsize == BUN_NONE) |
| 3040 | return GDK_FAIL; |
| 3041 | BAT *r1 = *r1p; |
| 3042 | BAT *r2 = r2p ? *r2p : NULL; |
| 3043 | |
| 3044 | lvals = (const char *) Tloc(l, 0); |
| 3045 | rvals = (const char *) Tloc(r, 0); |
| 3046 | assert(!r->tvarsized); |
| 3047 | lwidth = l->twidth; |
| 3048 | rwidth = r->twidth; |
| 3049 | |
| 3050 | assert(lvals != NULL); |
| 3051 | assert(rvals != NULL); |
| 3052 | |
| 3053 | r1->tkey = true; |
| 3054 | r1->tsorted = true; |
| 3055 | r1->trevsorted = true; |
| 3056 | if (r2) { |
| 3057 | r2->tkey = true; |
| 3058 | r2->tsorted = true; |
| 3059 | r2->trevsorted = true; |
| 3060 | } |
| 3061 | |
| 3062 | /* nested loop implementation for band join */ |
| 3063 | for (BUN li = 0; li < lcnt; li++) { |
| 3064 | lo = canditer_next(&lci); |
| 3065 | vl = FVALUE(l, lo - l->hseqbase); |
| 3066 | if (cmp(vl, nil) == 0) |
| 3067 | continue; |
| 3068 | nr = 0; |
| 3069 | canditer_reset(&rci); |
| 3070 | for (BUN ri = 0; ri < rcnt; ri++) { |
| 3071 | ro = canditer_next(&rci); |
| 3072 | vr = FVALUE(r, ro - r->hseqbase); |
| 3073 | switch (ATOMtype(l->ttype)) { |
| 3074 | case TYPE_bte: { |
| 3075 | if (is_bte_nil(*(const bte *) vr)) |
| 3076 | continue; |
| 3077 | sht v1 = (sht) *(const bte *) vr, v2; |
| 3078 | v2 = v1; |
| 3079 | v1 -= *(const bte *)c1; |
| 3080 | if (*(const bte *)vl <= v1 && |
| 3081 | (!li || *(const bte *)vl != v1)) |
| 3082 | continue; |
| 3083 | v2 += *(const bte *)c2; |
| 3084 | if (*(const bte *)vl >= v2 && |
| 3085 | (!hi || *(const bte *)vl != v2)) |
| 3086 | continue; |
| 3087 | break; |
| 3088 | } |
| 3089 | case TYPE_sht: { |
| 3090 | if (is_sht_nil(*(const sht *) vr)) |
| 3091 | continue; |
| 3092 | int v1 = (int) *(const sht *) vr, v2; |
| 3093 | v2 = v1; |
| 3094 | v1 -= *(const sht *)c1; |
| 3095 | if (*(const sht *)vl <= v1 && |
| 3096 | (!li || *(const sht *)vl != v1)) |
| 3097 | continue; |
| 3098 | v2 += *(const sht *)c2; |
| 3099 | if (*(const sht *)vl >= v2 && |
| 3100 | (!hi || *(const sht *)vl != v2)) |
| 3101 | continue; |
| 3102 | break; |
| 3103 | } |
| 3104 | case TYPE_int: { |
| 3105 | if (is_int_nil(*(const int *) vr)) |
| 3106 | continue; |
| 3107 | lng v1 = (lng) *(const int *) vr, v2; |
| 3108 | v2 = v1; |
| 3109 | v1 -= *(const int *)c1; |
| 3110 | if (*(const int *)vl <= v1 && |
| 3111 | (!li || *(const int *)vl != v1)) |
| 3112 | continue; |
| 3113 | v2 += *(const int *)c2; |
| 3114 | if (*(const int *)vl >= v2 && |
| 3115 | (!hi || *(const int *)vl != v2)) |
| 3116 | continue; |
| 3117 | break; |
| 3118 | } |
| 3119 | #ifdef HAVE_HGE |
| 3120 | case TYPE_lng: { |
| 3121 | if (is_lng_nil(*(const lng *) vr)) |
| 3122 | continue; |
| 3123 | hge v1 = (hge) *(const lng *) vr, v2; |
| 3124 | v2 = v1; |
| 3125 | v1 -= *(const lng *)c1; |
| 3126 | if (*(const lng *)vl <= v1 && |
| 3127 | (!li || *(const lng *)vl != v1)) |
| 3128 | continue; |
| 3129 | v2 += *(const lng *)c2; |
| 3130 | if (*(const lng *)vl >= v2 && |
| 3131 | (!hi || *(const lng *)vl != v2)) |
| 3132 | continue; |
| 3133 | break; |
| 3134 | } |
| 3135 | #else |
| 3136 | #ifdef HAVE___INT128 |
| 3137 | case TYPE_lng: { |
| 3138 | if (is_lng_nil(*(const lng *) vr)) |
| 3139 | continue; |
| 3140 | __int128 v1 = (__int128) *(const lng *) vr, v2; |
| 3141 | v2 = v1; |
| 3142 | v1 -= *(const lng *)c1; |
| 3143 | if (*(const lng *)vl <= v1 && |
| 3144 | (!li || *(const lng *)vl != v1)) |
| 3145 | continue; |
| 3146 | v2 += *(const lng *)c2; |
| 3147 | if (*(const lng *)vl >= v2 && |
| 3148 | (!hi || *(const lng *)vl != v2)) |
| 3149 | continue; |
| 3150 | break; |
| 3151 | } |
| 3152 | #else |
| 3153 | case TYPE_lng: { |
| 3154 | if (is_lng_nil(*(const lng *) vr)) |
| 3155 | continue; |
| 3156 | lng v1, v2; |
| 3157 | bool abort_on_error = true; |
| 3158 | SUB_WITH_CHECK(*(const lng *)vr, |
| 3159 | *(const lng *)c1, |
| 3160 | lng, v1, |
| 3161 | GDK_lng_max, |
| 3162 | do{if(*(const lng*)c1<0)goto nolmatch;else goto lmatch1;}while(false)); |
| 3163 | if (*(const lng *)vl <= v1 && |
| 3164 | (!li || *(const lng *)vl != v1)) |
| 3165 | continue; |
| 3166 | lmatch1: |
| 3167 | ADD_WITH_CHECK(*(const lng *)vr, |
| 3168 | *(const lng *)c2, |
| 3169 | lng, v2, |
| 3170 | GDK_lng_max, |
| 3171 | do{if(*(const lng*)c2>0)goto nolmatch;else goto lmatch2;}while(false)); |
| 3172 | if (*(const lng *)vl >= v2 && |
| 3173 | (!hi || *(const lng *)vl != v2)) |
| 3174 | continue; |
| 3175 | lmatch2: |
| 3176 | break; |
| 3177 | nolmatch: |
| 3178 | continue; |
| 3179 | } |
| 3180 | #endif |
| 3181 | #endif |
| 3182 | #ifdef HAVE_HGE |
| 3183 | case TYPE_hge: { |
| 3184 | if (is_hge_nil(*(const hge *) vr)) |
| 3185 | continue; |
| 3186 | hge v1, v2; |
| 3187 | bool abort_on_error = true; |
| 3188 | SUB_WITH_CHECK(*(const hge *)vr, |
| 3189 | *(const hge *)c1, |
| 3190 | hge, v1, |
| 3191 | GDK_hge_max, |
| 3192 | do{if(*(const hge*)c1<0)goto nohmatch;else goto hmatch1;}while(false)); |
| 3193 | if (*(const hge *)vl <= v1 && |
| 3194 | (!li || *(const hge *)vl != v1)) |
| 3195 | continue; |
| 3196 | hmatch1: |
| 3197 | ADD_WITH_CHECK(*(const hge *)vr, |
| 3198 | *(const hge *)c2, |
| 3199 | hge, v2, |
| 3200 | GDK_hge_max, |
| 3201 | do{if(*(const hge*)c2>0)goto nohmatch;else goto hmatch2;}while(false)); |
| 3202 | if (*(const hge *)vl >= v2 && |
| 3203 | (!hi || *(const hge *)vl != v2)) |
| 3204 | continue; |
| 3205 | hmatch2: |
| 3206 | break; |
| 3207 | nohmatch: |
| 3208 | continue; |
| 3209 | } |
| 3210 | #endif |
| 3211 | case TYPE_flt: { |
| 3212 | if (is_flt_nil(*(const flt *) vr)) |
| 3213 | continue; |
| 3214 | dbl v1 = (dbl) *(const flt *) vr, v2; |
| 3215 | v2 = v1; |
| 3216 | v1 -= *(const flt *)c1; |
| 3217 | if (*(const flt *)vl <= v1 && |
| 3218 | (!li || *(const flt *)vl != v1)) |
| 3219 | continue; |
| 3220 | v2 += *(const flt *)c2; |
| 3221 | if (*(const flt *)vl >= v2 && |
| 3222 | (!hi || *(const flt *)vl != v2)) |
| 3223 | continue; |
| 3224 | break; |
| 3225 | } |
| 3226 | case TYPE_dbl: { |
| 3227 | if (is_dbl_nil(*(const dbl *) vr)) |
| 3228 | continue; |
| 3229 | dbl v1, v2; |
| 3230 | bool abort_on_error = true; |
| 3231 | SUB_WITH_CHECK(*(const dbl *)vr, |
| 3232 | *(const dbl *)c1, |
| 3233 | dbl, v1, |
| 3234 | GDK_dbl_max, |
| 3235 | do{if(*(const dbl*)c1<0)goto nodmatch;else goto dmatch1;}while(false)); |
| 3236 | if (*(const dbl *)vl <= v1 && |
| 3237 | (!li || *(const dbl *)vl != v1)) |
| 3238 | continue; |
| 3239 | dmatch1: |
| 3240 | ADD_WITH_CHECK(*(const dbl *)vr, |
| 3241 | *(const dbl *)c2, |
| 3242 | dbl, v2, |
| 3243 | GDK_dbl_max, |
| 3244 | do{if(*(const dbl*)c2>0)goto nodmatch;else goto dmatch2;}while(false)); |
| 3245 | if (*(const dbl *)vl >= v2 && |
| 3246 | (!hi || *(const dbl *)vl != v2)) |
| 3247 | continue; |
| 3248 | dmatch2: |
| 3249 | break; |
| 3250 | nodmatch: |
| 3251 | continue; |
| 3252 | } |
| 3253 | } |
| 3254 | MAYBEEXTEND(1, &lci); |
| 3255 | if (BATcount(r1) > 0) { |
| 3256 | if (r2 && lastr + 1 != ro) |
| 3257 | r2->tseqbase = oid_nil; |
| 3258 | if (nr == 0) { |
| 3259 | r1->trevsorted = false; |
| 3260 | if (r2 == NULL) { |
| 3261 | /* nothing */ |
| 3262 | } else if (lastr > ro) { |
| 3263 | r2->tsorted = false; |
| 3264 | r2->tkey = false; |
| 3265 | } else if (lastr < ro) { |
| 3266 | r2->trevsorted = false; |
| 3267 | } else { |
| 3268 | r2->tkey = false; |
| 3269 | } |
| 3270 | } |
| 3271 | } |
| 3272 | APPEND(r1, lo); |
| 3273 | if (r2) { |
| 3274 | APPEND(r2, ro); |
| 3275 | } |
| 3276 | lastr = ro; |
| 3277 | nr++; |
| 3278 | } |
| 3279 | if (nr > 1) { |
| 3280 | r1->tkey = false; |
| 3281 | r1->tseqbase = oid_nil; |
| 3282 | if (r2) { |
| 3283 | r2->trevsorted = false; |
| 3284 | } |
| 3285 | } else if (nr == 0) { |
| 3286 | lskipped = BATcount(r1) > 0; |
| 3287 | } else if (lskipped) { |
| 3288 | r1->tseqbase = oid_nil; |
| 3289 | } |
| 3290 | } |
| 3291 | /* also set other bits of heap to correct value to indicate size */ |
| 3292 | BATsetcount(r1, BATcount(r1)); |
| 3293 | if (r2) { |
| 3294 | BATsetcount(r2, BATcount(r2)); |
| 3295 | assert(BATcount(r1) == BATcount(r2)); |
| 3296 | } |
| 3297 | if (BATcount(r1) > 0) { |
| 3298 | if (BATtdense(r1)) |
| 3299 | r1->tseqbase = ((oid *) r1->theap.base)[0]; |
| 3300 | if (r2 && BATtdense(r2)) |
| 3301 | r2->tseqbase = ((oid *) r2->theap.base)[0]; |
| 3302 | } else { |
| 3303 | r1->tseqbase = 0; |
| 3304 | if (r2) { |
| 3305 | r2->tseqbase = 0; |
| 3306 | } |
| 3307 | } |
| 3308 | ALGODEBUG fprintf(stderr, "#BATbandjoin(l=%s,r=%s)=("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 3309 | BATgetId(l), BATgetId(r), |
| 3310 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 3311 | GDKusec() - t0); |
| 3312 | return GDK_SUCCEED; |
| 3313 | |
| 3314 | bailout: |
| 3315 | BBPreclaim(r1); |
| 3316 | BBPreclaim(r2); |
| 3317 | return GDK_FAIL; |
| 3318 | } |
| 3319 | |
| 3320 | /* small ordered right, dense left, oid's only, do fetches */ |
| 3321 | static gdk_return |
| 3322 | fetchjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, |
| 3323 | struct canditer *restrict lci, struct canditer *restrict rci, |
| 3324 | const char *reason, lng t0) |
| 3325 | { |
| 3326 | oid lo = lci->seq - l->hseqbase + l->tseqbase, hi = lo + lci->ncand; |
| 3327 | BUN b, e, p; |
| 3328 | BAT *r1, *r2 = NULL; |
| 3329 | |
| 3330 | if (r->tsorted) { |
| 3331 | b = SORTfndfirst(r, &lo); |
| 3332 | e = SORTfndfirst(r, &hi); |
| 3333 | } else { |
| 3334 | assert(r->trevsorted); |
| 3335 | b = SORTfndlast(r, &hi); |
| 3336 | e = SORTfndlast(r, &lo); |
| 3337 | } |
| 3338 | if (b < rci->seq - r->hseqbase) |
| 3339 | b = rci->seq - r->hseqbase; |
| 3340 | if (e > rci->seq + rci->ncand - r->hseqbase) |
| 3341 | e = rci->seq + rci->ncand - r->hseqbase; |
| 3342 | if (e == b) { |
| 3343 | return nomatch(r1p, r2p, l, r, lci, |
| 3344 | false, false, "fetchjoin", t0); |
| 3345 | } |
| 3346 | r1 = COLnew(0, TYPE_oid, e - b, TRANSIENT); |
| 3347 | if (r1 == NULL) |
| 3348 | return GDK_FAIL; |
| 3349 | if (r2p) { |
| 3350 | if ((r2 = BATdense(0, r->hseqbase + b, e - b)) == NULL) { |
| 3351 | BBPreclaim(r1); |
| 3352 | return GDK_FAIL; |
| 3353 | } |
| 3354 | *r2p = r2; |
| 3355 | } |
| 3356 | *r1p = r1; |
| 3357 | oid *op = (oid *) Tloc(r1, 0); |
| 3358 | const oid *rp = (const oid *) Tloc(r, 0); |
| 3359 | for (p = b; p < e; p++) { |
| 3360 | *op++ = rp[p] + l->hseqbase - l->tseqbase; |
| 3361 | } |
| 3362 | BATsetcount(r1, e - b); |
| 3363 | r1->tkey = r->tkey; |
| 3364 | r1->tsorted = r->tsorted || e - b <= 1; |
| 3365 | r1->trevsorted = r->trevsorted || e - b <= 1; |
| 3366 | r1->tseqbase = e == b ? 0 : e - b == 1 ? *(const oid *)Tloc(r1, 0) : oid_nil; |
| 3367 | ALGODEBUG fprintf(stderr, "#%s: %s(l="ALGOBATFMT "," |
| 3368 | "r="ALGOBATFMT ",sl="ALGOOPTBATFMT "," |
| 3369 | "sr="ALGOOPTBATFMT ") %s " |
| 3370 | "-> ("ALGOBATFMT ","ALGOOPTBATFMT ") "LLFMT "us\n", |
| 3371 | MT_thread_getname(), __func__, |
| 3372 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 3373 | ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr), |
| 3374 | reason, |
| 3375 | ALGOBATPAR(r1), ALGOOPTBATPAR(r2), |
| 3376 | GDKusec() - t0); |
| 3377 | |
| 3378 | return GDK_SUCCEED; |
| 3379 | } |
| 3380 | |
| 3381 | |
| 3382 | /* Make the implementation choices for various left joins. */ |
| 3383 | static gdk_return |
| 3384 | leftjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, |
| 3385 | bool nil_matches, bool nil_on_miss, bool semi, bool only_misses, |
| 3386 | bool not_in, BUN estimate, const char *func, lng t0) |
| 3387 | { |
| 3388 | BUN lcnt, rcnt; |
| 3389 | struct canditer lci, rci; |
| 3390 | bool phash = false; |
| 3391 | |
| 3392 | /* only_misses implies left output only */ |
| 3393 | assert(!only_misses || r2p == NULL); |
| 3394 | /* if nil_on_miss is set, we really need a right output */ |
| 3395 | assert(!nil_on_miss || r2p != NULL); |
| 3396 | /* if not_in is set, then so is only_misses */ |
| 3397 | assert(!not_in || only_misses); |
| 3398 | *r1p = NULL; |
| 3399 | if (r2p) |
| 3400 | *r2p = NULL; |
| 3401 | if (joinparamcheck(l, r, NULL, sl, sr, func) != GDK_SUCCEED) |
| 3402 | return GDK_FAIL; |
| 3403 | |
| 3404 | lcnt = canditer_init(&lci, l, sl); |
| 3405 | rcnt = canditer_init(&rci, r, sr); |
| 3406 | |
| 3407 | if (lcnt == 0 || (!only_misses && !nil_on_miss && rcnt == 0)) { |
| 3408 | ALGODEBUG fprintf(stderr, "#%s(l="ALGOBATFMT "," |
| 3409 | "r="ALGOBATFMT ",sl="ALGOOPTBATFMT "," |
| 3410 | "sr="ALGOOPTBATFMT ",nil_matches=%d," |
| 3411 | "nil_on_miss=%d,semi=%d,only_misses=%d," |
| 3412 | "not_in=%d)\n", |
| 3413 | func, |
| 3414 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 3415 | ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr), |
| 3416 | nil_matches, nil_on_miss, semi, only_misses, |
| 3417 | not_in); |
| 3418 | return nomatch(r1p, r2p, l, r, &lci, |
| 3419 | nil_on_miss, only_misses, func, t0); |
| 3420 | } |
| 3421 | |
| 3422 | if (!nil_on_miss && !semi && !only_misses && !not_in && |
| 3423 | (lcnt == 1 || (BATordered(l) && BATordered_rev(l)) || |
| 3424 | (l->ttype == TYPE_void && is_oid_nil(l->tseqbase)))) { |
| 3425 | /* single value to join, use select */ |
| 3426 | return selectjoin(r1p, r2p, l, r, sl, sr, |
| 3427 | &lci, nil_matches, t0, false, func); |
| 3428 | } else if (BATtdense(r) && rci.tpe == cand_dense && |
| 3429 | lcnt > 0 && rcnt > 0) { |
| 3430 | /* use special implementation for dense right-hand side */ |
| 3431 | return mergejoin_void(r1p, r2p, l, r, sl, sr, &lci, &rci, |
| 3432 | nil_on_miss, only_misses, t0, false, |
| 3433 | func); |
| 3434 | } else if (BATtdense(l) |
| 3435 | && lci.tpe == cand_dense |
| 3436 | && rci.tpe == cand_dense |
| 3437 | && !semi |
| 3438 | && !nil_matches |
| 3439 | && !only_misses |
| 3440 | && !not_in |
| 3441 | /* && (rcnt * 1024) < lcnt */ |
| 3442 | && (BATordered(r) || BATordered_rev(r))) { |
| 3443 | assert(ATOMtype(l->ttype) == TYPE_oid); /* tdense */ |
| 3444 | return fetchjoin(r1p, r2p, l, r, sl, sr, &lci, &rci, func, t0); |
| 3445 | } else if ((BATordered(r) || BATordered_rev(r)) |
| 3446 | && (BATordered(l) |
| 3447 | || BATordered_rev(l) |
| 3448 | || BATtdense(r) |
| 3449 | || lcnt < 1024 |
| 3450 | || BATcount(r) * (Tsize(r) + (r->tvheap ? r->tvheap->size : 0) + 2 * sizeof(BUN)) > GDK_mem_maxsize / (GDKnr_threads ? GDKnr_threads : 1))) { |
| 3451 | return mergejoin(r1p, r2p, l, r, sl, sr, &lci, &rci, |
| 3452 | nil_matches, nil_on_miss, semi, only_misses, |
| 3453 | not_in, estimate, t0, false, func); |
| 3454 | } |
| 3455 | phash = sr == NULL && |
| 3456 | VIEWtparent(r) != 0 && |
| 3457 | BATcount(BBPquickdesc(VIEWtparent(r), false)) == BATcount(r); |
| 3458 | return hashjoin(r1p, r2p, l, r, sl, sr, &lci, &rci, |
| 3459 | nil_matches, nil_on_miss, semi, only_misses, |
| 3460 | not_in, estimate, t0, false, phash, func); |
| 3461 | } |
| 3462 | |
| 3463 | /* Perform an equi-join over l and r. Returns two new, aligned, bats |
| 3464 | * with the oids of matching tuples. The result is in the same order |
| 3465 | * as l (i.e. r1 is sorted). */ |
| 3466 | gdk_return |
| 3467 | BATleftjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, bool nil_matches, BUN estimate) |
| 3468 | { |
| 3469 | return leftjoin(r1p, r2p, l, r, sl, sr, nil_matches, |
| 3470 | false, false, false, false, estimate, "BATleftjoin", |
| 3471 | GDKdebug & ALGOMASK ? GDKusec() : 0); |
| 3472 | } |
| 3473 | |
| 3474 | /* Performs a left outer join over l and r. Returns two new, aligned, |
| 3475 | * bats with the oids of matching tuples, or the oid in the first |
| 3476 | * output bat and nil in the second output bat if the value in l does |
| 3477 | * not occur in r. The result is in the same order as l (i.e. r1 is |
| 3478 | * sorted). */ |
| 3479 | gdk_return |
| 3480 | BATouterjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, bool nil_matches, BUN estimate) |
| 3481 | { |
| 3482 | return leftjoin(r1p, r2p, l, r, sl, sr, nil_matches, |
| 3483 | true, false, false, false, estimate, "BATouterjoin", |
| 3484 | GDKdebug & ALGOMASK ? GDKusec() : 0); |
| 3485 | } |
| 3486 | |
| 3487 | /* Perform a semi-join over l and r. Returns one or two new, bats |
| 3488 | * with the oids of matching tuples. The result is in the same order |
| 3489 | * as l (i.e. r1 is sorted). If a single bat is returned, it is a |
| 3490 | * candidate list. */ |
| 3491 | gdk_return |
| 3492 | BATsemijoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, bool nil_matches, BUN estimate) |
| 3493 | { |
| 3494 | return leftjoin(r1p, r2p, l, r, sl, sr, nil_matches, |
| 3495 | false, true, false, false, estimate, "BATsemijoin", |
| 3496 | GDKdebug & ALGOMASK ? GDKusec() : 0); |
| 3497 | } |
| 3498 | |
| 3499 | /* Return a candidate list with the list of rows in l whose value also |
| 3500 | * occurs in r. This is just the left output of a semi-join. */ |
| 3501 | BAT * |
| 3502 | BATintersect(BAT *l, BAT *r, BAT *sl, BAT *sr, bool nil_matches, BUN estimate) |
| 3503 | { |
| 3504 | BAT *bn; |
| 3505 | |
| 3506 | if (leftjoin(&bn, NULL, l, r, sl, sr, nil_matches, |
| 3507 | false, true, false, false, estimate, "BATintersect", |
| 3508 | GDKdebug & ALGOMASK ? GDKusec() : 0) == GDK_SUCCEED) |
| 3509 | return virtualize(bn); |
| 3510 | return NULL; |
| 3511 | } |
| 3512 | |
| 3513 | /* Return the difference of l and r. The result is a BAT with the |
| 3514 | * oids of those values in l that do not occur in r. This is what you |
| 3515 | * might call an anti-semi-join. The result is a candidate list. */ |
| 3516 | BAT * |
| 3517 | BATdiff(BAT *l, BAT *r, BAT *sl, BAT *sr, bool nil_matches, bool not_in, |
| 3518 | BUN estimate) |
| 3519 | { |
| 3520 | BAT *bn; |
| 3521 | |
| 3522 | if (leftjoin(&bn, NULL, l, r, sl, sr, nil_matches, |
| 3523 | false, false, true, not_in, estimate, "BATdiff", |
| 3524 | GDKdebug & ALGOMASK ? GDKusec() : 0) == GDK_SUCCEED) |
| 3525 | return virtualize(bn); |
| 3526 | return NULL; |
| 3527 | } |
| 3528 | |
| 3529 | gdk_return |
| 3530 | BATthetajoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, int op, bool nil_matches, BUN estimate) |
| 3531 | { |
| 3532 | int opcode = 0; |
| 3533 | lng t0 = 0; |
| 3534 | |
| 3535 | /* encode operator as a bit mask into opcode */ |
| 3536 | switch (op) { |
| 3537 | case JOIN_EQ: |
| 3538 | return BATjoin(r1p, r2p, l, r, sl, sr, nil_matches, estimate); |
| 3539 | case JOIN_NE: |
| 3540 | opcode = MASK_NE; |
| 3541 | break; |
| 3542 | case JOIN_LT: |
| 3543 | opcode = MASK_LT; |
| 3544 | break; |
| 3545 | case JOIN_LE: |
| 3546 | opcode = MASK_LE; |
| 3547 | break; |
| 3548 | case JOIN_GT: |
| 3549 | opcode = MASK_GT; |
| 3550 | break; |
| 3551 | case JOIN_GE: |
| 3552 | opcode = MASK_GE; |
| 3553 | break; |
| 3554 | default: |
| 3555 | GDKerror("BATthetajoin: unknown operator %d.\n", op); |
| 3556 | return GDK_FAIL; |
| 3557 | } |
| 3558 | |
| 3559 | ALGODEBUG t0 = GDKusec(); |
| 3560 | *r1p = NULL; |
| 3561 | if (r2p) { |
| 3562 | *r2p = NULL; |
| 3563 | } |
| 3564 | if (joinparamcheck(l, r, NULL, sl, sr, "BATthetajoin") != GDK_SUCCEED) |
| 3565 | return GDK_FAIL; |
| 3566 | |
| 3567 | return thetajoin(r1p, r2p, l, r, sl, sr, opcode, estimate, t0); |
| 3568 | } |
| 3569 | |
| 3570 | gdk_return |
| 3571 | BATjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, bool nil_matches, BUN estimate) |
| 3572 | { |
| 3573 | struct canditer lci, rci; |
| 3574 | BUN lcnt, rcnt; |
| 3575 | BUN lsize, rsize; |
| 3576 | bool lhash = false, rhash = false; |
| 3577 | bool plhash = false, prhash = false; |
| 3578 | BUN lslots = 0, rslots = 0; |
| 3579 | bool swap; |
| 3580 | bat parent; |
| 3581 | size_t mem_size; |
| 3582 | lng t0 = 0; |
| 3583 | const char *reason = ""; |
| 3584 | |
| 3585 | ALGODEBUG t0 = GDKusec(); |
| 3586 | |
| 3587 | if ((parent = VIEWtparent(l)) != 0) { |
| 3588 | BAT *b = BBPdescriptor(parent); |
| 3589 | if (l->hseqbase == b->hseqbase && |
| 3590 | BATcount(l) == BATcount(b)) |
| 3591 | l = b; |
| 3592 | } |
| 3593 | if ((parent = VIEWtparent(r)) != 0) { |
| 3594 | BAT *b = BBPdescriptor(parent); |
| 3595 | if (r->hseqbase == b->hseqbase && |
| 3596 | BATcount(r) == BATcount(b)) |
| 3597 | r = b; |
| 3598 | } |
| 3599 | |
| 3600 | lcnt = canditer_init(&lci, l, sl); |
| 3601 | rcnt = canditer_init(&rci, r, sr); |
| 3602 | |
| 3603 | *r1p = NULL; |
| 3604 | if (r2p) { |
| 3605 | *r2p = NULL; |
| 3606 | } |
| 3607 | if (joinparamcheck(l, r, NULL, sl, sr, "BATjoin") != GDK_SUCCEED) |
| 3608 | return GDK_FAIL; |
| 3609 | |
| 3610 | if (lcnt == 0 || rcnt == 0) { |
| 3611 | ALGODEBUG fprintf(stderr, "#BATjoin(l="ALGOBATFMT "," |
| 3612 | "r="ALGOBATFMT ",sl="ALGOOPTBATFMT "," |
| 3613 | "sr="ALGOOPTBATFMT ",nil_matches=%d)\n", |
| 3614 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 3615 | ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr), |
| 3616 | nil_matches); |
| 3617 | return nomatch(r1p, r2p, l, r, &lci, |
| 3618 | false, false, "BATjoin", t0); |
| 3619 | } |
| 3620 | |
| 3621 | swap = false; |
| 3622 | |
| 3623 | /* some statistics to help us decide */ |
| 3624 | lsize = (BUN) (BATcount(l) * (Tsize(l)) + (l->tvheap ? l->tvheap->size : 0) + 2 * sizeof(BUN)); |
| 3625 | rsize = (BUN) (BATcount(r) * (Tsize(r)) + (r->tvheap ? r->tvheap->size : 0) + 2 * sizeof(BUN)); |
| 3626 | mem_size = GDK_mem_maxsize / (GDKnr_threads ? GDKnr_threads : 1); |
| 3627 | |
| 3628 | if (lcnt == 1 || (BATordered(l) && BATordered_rev(l)) || (l->ttype == TYPE_void && is_oid_nil(l->tseqbase))) { |
| 3629 | /* single value to join, use select */ |
| 3630 | return selectjoin(r1p, r2p, l, r, sl, sr, |
| 3631 | &lci, nil_matches, t0, false, __func__); |
| 3632 | } else if (r2p != NULL && (rcnt == 1 || (BATordered(r) && BATordered_rev(r)) || (r->ttype == TYPE_void && is_oid_nil(r->tseqbase)))) { |
| 3633 | /* single value to join, use select */ |
| 3634 | return selectjoin(r2p, r1p, r, l, sr, sl, |
| 3635 | &rci, nil_matches, t0, true, __func__); |
| 3636 | } else if (BATtdense(r) && rci.tpe == cand_dense) { |
| 3637 | /* use special implementation for dense right-hand side */ |
| 3638 | return mergejoin_void(r1p, r2p, l, r, sl, sr, &lci, &rci, |
| 3639 | false, false, t0, false, __func__); |
| 3640 | } else if (r2p && BATtdense(l) && lci.tpe == cand_dense) { |
| 3641 | /* use special implementation for dense right-hand side */ |
| 3642 | return mergejoin_void(r2p, r1p, r, l, sr, sl, &rci, &lci, |
| 3643 | false, false, t0, true, __func__); |
| 3644 | } else if ((BATordered(l) || BATordered_rev(l)) && |
| 3645 | (BATordered(r) || BATordered_rev(r))) { |
| 3646 | /* both sorted */ |
| 3647 | return mergejoin(r1p, r2p, l, r, sl, sr, &lci, &rci, |
| 3648 | nil_matches, false, false, false, false, |
| 3649 | estimate, t0, false, __func__); |
| 3650 | } |
| 3651 | if (sl == NULL) { |
| 3652 | lhash = BATcheckhash(l); |
| 3653 | if (lhash) { |
| 3654 | lslots = ((size_t *) l->thash->heap.base)[5]; |
| 3655 | } else if ((parent = VIEWtparent(l)) != 0) { |
| 3656 | BAT *b = BBPdescriptor(parent); |
| 3657 | /* use hash on parent if the average chain |
| 3658 | * length times the number of required probes |
| 3659 | * is less than the cost for creating and |
| 3660 | * probing a new hash on the view */ |
| 3661 | if (BATcheckhash(b)) { |
| 3662 | lslots = ((size_t *) b->thash->heap.base)[5]; |
| 3663 | lhash = (BATcount(b) == BATcount(l) || |
| 3664 | BATcount(b) / lslots * rcnt < lcnt + rcnt); |
| 3665 | } |
| 3666 | plhash = lhash; |
| 3667 | } |
| 3668 | } else if (BATtdense(sl) && BATcheckhash(l)) { |
| 3669 | lslots = ((size_t *) l->thash->heap.base)[5]; |
| 3670 | lhash = BATcount(l) / lslots * rcnt < lcnt + rcnt; |
| 3671 | } |
| 3672 | if (sr == NULL) { |
| 3673 | rhash = BATcheckhash(r); |
| 3674 | if (rhash) { |
| 3675 | rslots = ((size_t *) r->thash->heap.base)[5]; |
| 3676 | } else if ((parent = VIEWtparent(r)) != 0) { |
| 3677 | BAT *b = BBPdescriptor(parent); |
| 3678 | /* use hash on parent if the average chain |
| 3679 | * length times the number of required probes |
| 3680 | * is less than the cost for creating and |
| 3681 | * probing a new hash on the view */ |
| 3682 | if (BATcheckhash(b)) { |
| 3683 | rslots = ((size_t *) b->thash->heap.base)[5]; |
| 3684 | rhash = (BATcount(b) == BATcount(r) || |
| 3685 | BATcount(b) / rslots * lcnt < lcnt + rcnt); |
| 3686 | } |
| 3687 | prhash = rhash; |
| 3688 | } |
| 3689 | } else if (BATtdense(sr) && BATcheckhash(r)) { |
| 3690 | rslots = ((size_t *) r->thash->heap.base)[5]; |
| 3691 | rhash = BATcount(r) / rslots * rcnt < lcnt + rcnt; |
| 3692 | } |
| 3693 | if (lhash && rhash) { |
| 3694 | if (lcnt == lslots && rcnt == rslots) { |
| 3695 | /* both perfect hashes, smallest on right */ |
| 3696 | swap = r2p && lcnt < rcnt; |
| 3697 | } else if (r2p && lcnt == lslots) { |
| 3698 | /* left is perfect (right isn't): swap */ |
| 3699 | swap = true; |
| 3700 | } else if (rcnt != rslots) { |
| 3701 | /* neither is perfect, shortest chains on right */ |
| 3702 | swap = r2p && lcnt / lslots < rcnt / rslots; |
| 3703 | } /* else: right is perfect */ |
| 3704 | reason = "both have hash"; |
| 3705 | } else if (r2p && lhash) { |
| 3706 | /* only left has hash, swap */ |
| 3707 | swap = true; |
| 3708 | reason = "left has hash"; |
| 3709 | } else if (rhash) { |
| 3710 | /* only right has hash, don't swap */ |
| 3711 | swap = false; |
| 3712 | reason = "right has hash"; |
| 3713 | } else if (r2p && |
| 3714 | (BATordered(l) || BATordered_rev(l)) && |
| 3715 | (BATtvoid(l) || rcnt < 1024 || MIN(lsize, rsize) > mem_size)) { |
| 3716 | /* only left is sorted, swap; but only if right is |
| 3717 | * "large" and the smaller of the two isn't too large |
| 3718 | * (i.e. prefer hash over binary search, but only if |
| 3719 | * the hash table doesn't cause thrashing) */ |
| 3720 | return mergejoin(r2p, r1p, r, l, sr, sl, &rci, &lci, |
| 3721 | nil_matches, false, false, false, false, |
| 3722 | estimate, t0, true, __func__); |
| 3723 | } else if ((BATordered(r) || BATordered_rev(r)) && |
| 3724 | (BATtvoid(r) || lcnt < 1024 || MIN(lsize, rsize) > mem_size)) { |
| 3725 | /* only right is sorted, don't swap; but only if left |
| 3726 | * is "large" and the smaller of the two isn't too |
| 3727 | * large (i.e. prefer hash over binary search, but |
| 3728 | * only if the hash table doesn't cause thrashing) */ |
| 3729 | return mergejoin(r1p, r2p, l, r, sl, sr, &lci, &rci, |
| 3730 | nil_matches, false, false, false, false, |
| 3731 | estimate, t0, false, __func__); |
| 3732 | } else if (r2p && !l->batTransient && r->batTransient) { |
| 3733 | /* l is persistent and r is not, create hash on l |
| 3734 | * since it may be reused */ |
| 3735 | swap = true; |
| 3736 | reason = "left is persistent"; |
| 3737 | } else if (l->batTransient && !r->batTransient) { |
| 3738 | /* l is not persistent but r is, create hash on r |
| 3739 | * since it may be reused */ |
| 3740 | /* nothing */; |
| 3741 | reason = "right is persistent"; |
| 3742 | } else if (r2p && lcnt < rcnt) { |
| 3743 | /* no hashes, not sorted, create hash on smallest BAT */ |
| 3744 | swap = true; |
| 3745 | reason = "left is smaller"; |
| 3746 | } |
| 3747 | if (swap) { |
| 3748 | assert(r2p); |
| 3749 | return hashjoin(r2p, r1p, r, l, sr, sl, &rci, &lci, |
| 3750 | nil_matches, false, false, false, false, |
| 3751 | estimate, t0, true, plhash, reason); |
| 3752 | } else { |
| 3753 | return hashjoin(r1p, r2p, l, r, sl, sr, &lci, &rci, |
| 3754 | nil_matches, false, false, false, false, |
| 3755 | estimate, t0, false, prhash, reason); |
| 3756 | } |
| 3757 | } |
| 3758 | |
| 3759 | gdk_return |
| 3760 | BATbandjoin(BAT **r1p, BAT **r2p, BAT *l, BAT *r, BAT *sl, BAT *sr, |
| 3761 | const void *c1, const void *c2, bool li, bool hi, BUN estimate) |
| 3762 | { |
| 3763 | lng t0 = 0; |
| 3764 | |
| 3765 | ALGODEBUG t0 = GDKusec(); |
| 3766 | |
| 3767 | ALGODEBUG fprintf(stderr, "#BATbandjoin(" |
| 3768 | "l="ALGOBATFMT ",r="ALGOBATFMT "," |
| 3769 | "sl="ALGOOPTBATFMT ",sr="ALGOOPTBATFMT ")\n", |
| 3770 | ALGOBATPAR(l), ALGOBATPAR(r), |
| 3771 | ALGOOPTBATPAR(sl), ALGOOPTBATPAR(sr)); |
| 3772 | |
| 3773 | *r1p = NULL; |
| 3774 | if (r2p) { |
| 3775 | *r2p = NULL; |
| 3776 | } |
| 3777 | if (joinparamcheck(l, r, NULL, sl, sr, "BATbandjoin") != GDK_SUCCEED) |
| 3778 | return GDK_FAIL; |
| 3779 | return bandjoin(r1p, r2p, l, r, sl, sr, c1, c2, li, hi, estimate, t0); |
| 3780 | } |
| 3781 | |
| 3782 | gdk_return |
| 3783 | BATrangejoin(BAT **r1p, BAT **r2p, BAT *l, BAT *rl, BAT *rh, |
| 3784 | BAT *sl, BAT *sr, bool li, bool hi, bool anti, bool symmetric, |
| 3785 | BUN estimate) |
| 3786 | { |
| 3787 | struct canditer lci, rci; |
| 3788 | BAT *r1, *r2; |
| 3789 | BUN maxsize; |
| 3790 | lng t0 = 0; |
| 3791 | |
| 3792 | ALGODEBUG t0 = GDKusec(); |
| 3793 | *r1p = NULL; |
| 3794 | if (r2p) { |
| 3795 | *r2p = NULL; |
| 3796 | } |
| 3797 | if (joinparamcheck(l, rl, rh, sl, sr, "BATrangejoin") != GDK_SUCCEED) |
| 3798 | return GDK_FAIL; |
| 3799 | if (canditer_init(&lci, l, sl) == 0 || |
| 3800 | canditer_init(&rci, rl, sr) == 0 || |
| 3801 | (l->ttype == TYPE_void && is_oid_nil(l->tseqbase)) || |
| 3802 | ((rl->ttype == TYPE_void && is_oid_nil(rl->tseqbase)) && |
| 3803 | (rh->ttype == TYPE_void && is_oid_nil(rh->tseqbase)))) { |
| 3804 | /* trivial: empty input */ |
| 3805 | return nomatch(r1p, r2p, l, rl, &lci, false, false, |
| 3806 | __func__, t0); |
| 3807 | } |
| 3808 | if (rl->ttype == TYPE_void && is_oid_nil(rl->tseqbase)) { |
| 3809 | if (!anti) |
| 3810 | return nomatch(r1p, r2p, l, rl, &lci, false, false, |
| 3811 | __func__, t0); |
| 3812 | return thetajoin(r1p, r2p, l, rh, sl, sr, MASK_GT, estimate, t0); |
| 3813 | } |
| 3814 | if (rh->ttype == TYPE_void && is_oid_nil(rh->tseqbase)) { |
| 3815 | if (!anti) |
| 3816 | return nomatch(r1p, r2p, l, rl, &lci, false, false, |
| 3817 | __func__, t0); |
| 3818 | return thetajoin(r1p, r2p, l, rl, sl, sr, MASK_LT, estimate, t0); |
| 3819 | } |
| 3820 | |
| 3821 | if ((maxsize = joininitresults(&r1, r2p ? &r2 : NULL, sl ? BATcount(sl) : BATcount(l), sr ? BATcount(sr) : BATcount(rl), false, false, false, false, false, estimate)) == BUN_NONE) |
| 3822 | return GDK_FAIL; |
| 3823 | *r1p = r1; |
| 3824 | if (r2p) { |
| 3825 | *r2p = r2; |
| 3826 | } |
| 3827 | if (maxsize == 0) |
| 3828 | return GDK_SUCCEED; |
| 3829 | |
| 3830 | /* note, the rangejoin implementation is in gdk_select.c since |
| 3831 | * it uses the imprints code there */ |
| 3832 | return rangejoin(r1, r2, l, rl, rh, &lci, &rci, li, hi, anti, symmetric, maxsize); |
| 3833 | } |
| 3834 |
Generated on 2019-Nov-27 from project MonetDB revision 11.36
Powered by 
Code Browser 2.1
Generator usage only permitted with license.