1 | /* |
2 | ** 2001 September 15 |
3 | ** |
4 | ** The author disclaims copyright to this source code. In place of |
5 | ** a legal notice, here is a blessing: |
6 | ** |
7 | ** May you do good and not evil. |
8 | ** May you find forgiveness for yourself and forgive others. |
9 | ** May you share freely, never taking more than you give. |
10 | ** |
11 | ************************************************************************* |
12 | ** This module contains C code that generates VDBE code used to process |
13 | ** the WHERE clause of SQL statements. This module is responsible for |
14 | ** generating the code that loops through a table looking for applicable |
15 | ** rows. Indices are selected and used to speed the search when doing |
16 | ** so is applicable. Because this module is responsible for selecting |
17 | ** indices, you might also think of this module as the "query optimizer". |
18 | */ |
19 | #include "sqliteInt.h" |
20 | #include "whereInt.h" |
21 | |
22 | /* |
23 | ** Extra information appended to the end of sqlite3_index_info but not |
24 | ** visible to the xBestIndex function, at least not directly. The |
25 | ** sqlite3_vtab_collation() interface knows how to reach it, however. |
26 | ** |
27 | ** This object is not an API and can be changed from one release to the |
28 | ** next. As long as allocateIndexInfo() and sqlite3_vtab_collation() |
29 | ** agree on the structure, all will be well. |
30 | */ |
31 | typedef struct HiddenIndexInfo HiddenIndexInfo; |
32 | struct HiddenIndexInfo { |
33 | WhereClause *pWC; /* The Where clause being analyzed */ |
34 | Parse *pParse; /* The parsing context */ |
35 | int eDistinct; /* Value to return from sqlite3_vtab_distinct() */ |
36 | u32 mIn; /* Mask of terms that are <col> IN (...) */ |
37 | u32 mHandleIn; /* Terms that vtab will handle as <col> IN (...) */ |
38 | sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST |
39 | ** because extra space is allocated to hold up |
40 | ** to nTerm such values */ |
41 | }; |
42 | |
43 | /* Forward declaration of methods */ |
44 | static int whereLoopResize(sqlite3*, WhereLoop*, int); |
45 | |
46 | /* |
47 | ** Return the estimated number of output rows from a WHERE clause |
48 | */ |
49 | LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ |
50 | return pWInfo->nRowOut; |
51 | } |
52 | |
53 | /* |
54 | ** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this |
55 | ** WHERE clause returns outputs for DISTINCT processing. |
56 | */ |
57 | int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ |
58 | return pWInfo->eDistinct; |
59 | } |
60 | |
61 | /* |
62 | ** Return the number of ORDER BY terms that are satisfied by the |
63 | ** WHERE clause. A return of 0 means that the output must be |
64 | ** completely sorted. A return equal to the number of ORDER BY |
65 | ** terms means that no sorting is needed at all. A return that |
66 | ** is positive but less than the number of ORDER BY terms means that |
67 | ** block sorting is required. |
68 | */ |
69 | int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ |
70 | return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat; |
71 | } |
72 | |
73 | /* |
74 | ** In the ORDER BY LIMIT optimization, if the inner-most loop is known |
75 | ** to emit rows in increasing order, and if the last row emitted by the |
76 | ** inner-most loop did not fit within the sorter, then we can skip all |
77 | ** subsequent rows for the current iteration of the inner loop (because they |
78 | ** will not fit in the sorter either) and continue with the second inner |
79 | ** loop - the loop immediately outside the inner-most. |
80 | ** |
81 | ** When a row does not fit in the sorter (because the sorter already |
82 | ** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the |
83 | ** label returned by this function. |
84 | ** |
85 | ** If the ORDER BY LIMIT optimization applies, the jump destination should |
86 | ** be the continuation for the second-inner-most loop. If the ORDER BY |
87 | ** LIMIT optimization does not apply, then the jump destination should |
88 | ** be the continuation for the inner-most loop. |
89 | ** |
90 | ** It is always safe for this routine to return the continuation of the |
91 | ** inner-most loop, in the sense that a correct answer will result. |
92 | ** Returning the continuation the second inner loop is an optimization |
93 | ** that might make the code run a little faster, but should not change |
94 | ** the final answer. |
95 | */ |
96 | int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ |
97 | WhereLevel *pInner; |
98 | if( !pWInfo->bOrderedInnerLoop ){ |
99 | /* The ORDER BY LIMIT optimization does not apply. Jump to the |
100 | ** continuation of the inner-most loop. */ |
101 | return pWInfo->iContinue; |
102 | } |
103 | pInner = &pWInfo->a[pWInfo->nLevel-1]; |
104 | assert( pInner->addrNxt!=0 ); |
105 | return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt; |
106 | } |
107 | |
108 | /* |
109 | ** While generating code for the min/max optimization, after handling |
110 | ** the aggregate-step call to min() or max(), check to see if any |
111 | ** additional looping is required. If the output order is such that |
112 | ** we are certain that the correct answer has already been found, then |
113 | ** code an OP_Goto to by pass subsequent processing. |
114 | ** |
115 | ** Any extra OP_Goto that is coded here is an optimization. The |
116 | ** correct answer should be obtained regardless. This OP_Goto just |
117 | ** makes the answer appear faster. |
118 | */ |
119 | void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){ |
120 | WhereLevel *pInner; |
121 | int i; |
122 | if( !pWInfo->bOrderedInnerLoop ) return; |
123 | if( pWInfo->nOBSat==0 ) return; |
124 | for(i=pWInfo->nLevel-1; i>=0; i--){ |
125 | pInner = &pWInfo->a[i]; |
126 | if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){ |
127 | sqlite3VdbeGoto(v, pInner->addrNxt); |
128 | return; |
129 | } |
130 | } |
131 | sqlite3VdbeGoto(v, pWInfo->iBreak); |
132 | } |
133 | |
134 | /* |
135 | ** Return the VDBE address or label to jump to in order to continue |
136 | ** immediately with the next row of a WHERE clause. |
137 | */ |
138 | int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ |
139 | assert( pWInfo->iContinue!=0 ); |
140 | return pWInfo->iContinue; |
141 | } |
142 | |
143 | /* |
144 | ** Return the VDBE address or label to jump to in order to break |
145 | ** out of a WHERE loop. |
146 | */ |
147 | int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ |
148 | return pWInfo->iBreak; |
149 | } |
150 | |
151 | /* |
152 | ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to |
153 | ** operate directly on the rowids returned by a WHERE clause. Return |
154 | ** ONEPASS_SINGLE (1) if the statement can operation directly because only |
155 | ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass |
156 | ** optimization can be used on multiple |
157 | ** |
158 | ** If the ONEPASS optimization is used (if this routine returns true) |
159 | ** then also write the indices of open cursors used by ONEPASS |
160 | ** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data |
161 | ** table and iaCur[1] gets the cursor used by an auxiliary index. |
162 | ** Either value may be -1, indicating that cursor is not used. |
163 | ** Any cursors returned will have been opened for writing. |
164 | ** |
165 | ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is |
166 | ** unable to use the ONEPASS optimization. |
167 | */ |
168 | int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ |
169 | memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); |
170 | #ifdef WHERETRACE_ENABLED |
171 | if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ |
172 | sqlite3DebugPrintf("%s cursors: %d %d\n" , |
173 | pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI" , |
174 | aiCur[0], aiCur[1]); |
175 | } |
176 | #endif |
177 | return pWInfo->eOnePass; |
178 | } |
179 | |
180 | /* |
181 | ** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move |
182 | ** the data cursor to the row selected by the index cursor. |
183 | */ |
184 | int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){ |
185 | return pWInfo->bDeferredSeek; |
186 | } |
187 | |
188 | /* |
189 | ** Move the content of pSrc into pDest |
190 | */ |
191 | static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ |
192 | pDest->n = pSrc->n; |
193 | memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); |
194 | } |
195 | |
196 | /* |
197 | ** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. |
198 | ** |
199 | ** The new entry might overwrite an existing entry, or it might be |
200 | ** appended, or it might be discarded. Do whatever is the right thing |
201 | ** so that pSet keeps the N_OR_COST best entries seen so far. |
202 | */ |
203 | static int whereOrInsert( |
204 | WhereOrSet *pSet, /* The WhereOrSet to be updated */ |
205 | Bitmask prereq, /* Prerequisites of the new entry */ |
206 | LogEst rRun, /* Run-cost of the new entry */ |
207 | LogEst nOut /* Number of outputs for the new entry */ |
208 | ){ |
209 | u16 i; |
210 | WhereOrCost *p; |
211 | for(i=pSet->n, p=pSet->a; i>0; i--, p++){ |
212 | if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ |
213 | goto whereOrInsert_done; |
214 | } |
215 | if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ |
216 | return 0; |
217 | } |
218 | } |
219 | if( pSet->n<N_OR_COST ){ |
220 | p = &pSet->a[pSet->n++]; |
221 | p->nOut = nOut; |
222 | }else{ |
223 | p = pSet->a; |
224 | for(i=1; i<pSet->n; i++){ |
225 | if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; |
226 | } |
227 | if( p->rRun<=rRun ) return 0; |
228 | } |
229 | whereOrInsert_done: |
230 | p->prereq = prereq; |
231 | p->rRun = rRun; |
232 | if( p->nOut>nOut ) p->nOut = nOut; |
233 | return 1; |
234 | } |
235 | |
236 | /* |
237 | ** Return the bitmask for the given cursor number. Return 0 if |
238 | ** iCursor is not in the set. |
239 | */ |
240 | Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ |
241 | int i; |
242 | assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); |
243 | assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 ); |
244 | assert( iCursor>=-1 ); |
245 | if( pMaskSet->ix[0]==iCursor ){ |
246 | return 1; |
247 | } |
248 | for(i=1; i<pMaskSet->n; i++){ |
249 | if( pMaskSet->ix[i]==iCursor ){ |
250 | return MASKBIT(i); |
251 | } |
252 | } |
253 | return 0; |
254 | } |
255 | |
256 | /* Allocate memory that is automatically freed when pWInfo is freed. |
257 | */ |
258 | void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){ |
259 | WhereMemBlock *pBlock; |
260 | pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock)); |
261 | if( pBlock ){ |
262 | pBlock->pNext = pWInfo->pMemToFree; |
263 | pBlock->sz = nByte; |
264 | pWInfo->pMemToFree = pBlock; |
265 | pBlock++; |
266 | } |
267 | return (void*)pBlock; |
268 | } |
269 | void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){ |
270 | void *pNew = sqlite3WhereMalloc(pWInfo, nByte); |
271 | if( pNew && pOld ){ |
272 | WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld; |
273 | pOldBlk--; |
274 | assert( pOldBlk->sz<nByte ); |
275 | memcpy(pNew, pOld, pOldBlk->sz); |
276 | } |
277 | return pNew; |
278 | } |
279 | |
280 | /* |
281 | ** Create a new mask for cursor iCursor. |
282 | ** |
283 | ** There is one cursor per table in the FROM clause. The number of |
284 | ** tables in the FROM clause is limited by a test early in the |
285 | ** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] |
286 | ** array will never overflow. |
287 | */ |
288 | static void createMask(WhereMaskSet *pMaskSet, int iCursor){ |
289 | assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); |
290 | pMaskSet->ix[pMaskSet->n++] = iCursor; |
291 | } |
292 | |
293 | /* |
294 | ** If the right-hand branch of the expression is a TK_COLUMN, then return |
295 | ** a pointer to the right-hand branch. Otherwise, return NULL. |
296 | */ |
297 | static Expr *whereRightSubexprIsColumn(Expr *p){ |
298 | p = sqlite3ExprSkipCollateAndLikely(p->pRight); |
299 | if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ |
300 | return p; |
301 | } |
302 | return 0; |
303 | } |
304 | |
305 | /* |
306 | ** Advance to the next WhereTerm that matches according to the criteria |
307 | ** established when the pScan object was initialized by whereScanInit(). |
308 | ** Return NULL if there are no more matching WhereTerms. |
309 | */ |
310 | static WhereTerm *whereScanNext(WhereScan *pScan){ |
311 | int iCur; /* The cursor on the LHS of the term */ |
312 | i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ |
313 | Expr *pX; /* An expression being tested */ |
314 | WhereClause *pWC; /* Shorthand for pScan->pWC */ |
315 | WhereTerm *pTerm; /* The term being tested */ |
316 | int k = pScan->k; /* Where to start scanning */ |
317 | |
318 | assert( pScan->iEquiv<=pScan->nEquiv ); |
319 | pWC = pScan->pWC; |
320 | while(1){ |
321 | iColumn = pScan->aiColumn[pScan->iEquiv-1]; |
322 | iCur = pScan->aiCur[pScan->iEquiv-1]; |
323 | assert( pWC!=0 ); |
324 | assert( iCur>=0 ); |
325 | do{ |
326 | for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){ |
327 | assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 ); |
328 | if( pTerm->leftCursor==iCur |
329 | && pTerm->u.x.leftColumn==iColumn |
330 | && (iColumn!=XN_EXPR |
331 | || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft, |
332 | pScan->pIdxExpr,iCur)==0) |
333 | && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON)) |
334 | ){ |
335 | if( (pTerm->eOperator & WO_EQUIV)!=0 |
336 | && pScan->nEquiv<ArraySize(pScan->aiCur) |
337 | && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0 |
338 | ){ |
339 | int j; |
340 | for(j=0; j<pScan->nEquiv; j++){ |
341 | if( pScan->aiCur[j]==pX->iTable |
342 | && pScan->aiColumn[j]==pX->iColumn ){ |
343 | break; |
344 | } |
345 | } |
346 | if( j==pScan->nEquiv ){ |
347 | pScan->aiCur[j] = pX->iTable; |
348 | pScan->aiColumn[j] = pX->iColumn; |
349 | pScan->nEquiv++; |
350 | } |
351 | } |
352 | if( (pTerm->eOperator & pScan->opMask)!=0 ){ |
353 | /* Verify the affinity and collating sequence match */ |
354 | if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ |
355 | CollSeq *pColl; |
356 | Parse *pParse = pWC->pWInfo->pParse; |
357 | pX = pTerm->pExpr; |
358 | if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ |
359 | continue; |
360 | } |
361 | assert(pX->pLeft); |
362 | pColl = sqlite3ExprCompareCollSeq(pParse, pX); |
363 | if( pColl==0 ) pColl = pParse->db->pDfltColl; |
364 | if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ |
365 | continue; |
366 | } |
367 | } |
368 | if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 |
369 | && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0)) |
370 | && pX->op==TK_COLUMN |
371 | && pX->iTable==pScan->aiCur[0] |
372 | && pX->iColumn==pScan->aiColumn[0] |
373 | ){ |
374 | testcase( pTerm->eOperator & WO_IS ); |
375 | continue; |
376 | } |
377 | pScan->pWC = pWC; |
378 | pScan->k = k+1; |
379 | #ifdef WHERETRACE_ENABLED |
380 | if( sqlite3WhereTrace & 0x20000 ){ |
381 | int ii; |
382 | sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d" , |
383 | pTerm, pScan->nEquiv); |
384 | for(ii=0; ii<pScan->nEquiv; ii++){ |
385 | sqlite3DebugPrintf(" {%d:%d}" , |
386 | pScan->aiCur[ii], pScan->aiColumn[ii]); |
387 | } |
388 | sqlite3DebugPrintf("\n" ); |
389 | } |
390 | #endif |
391 | return pTerm; |
392 | } |
393 | } |
394 | } |
395 | pWC = pWC->pOuter; |
396 | k = 0; |
397 | }while( pWC!=0 ); |
398 | if( pScan->iEquiv>=pScan->nEquiv ) break; |
399 | pWC = pScan->pOrigWC; |
400 | k = 0; |
401 | pScan->iEquiv++; |
402 | } |
403 | return 0; |
404 | } |
405 | |
406 | /* |
407 | ** This is whereScanInit() for the case of an index on an expression. |
408 | ** It is factored out into a separate tail-recursion subroutine so that |
409 | ** the normal whereScanInit() routine, which is a high-runner, does not |
410 | ** need to push registers onto the stack as part of its prologue. |
411 | */ |
412 | static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){ |
413 | pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr); |
414 | return whereScanNext(pScan); |
415 | } |
416 | |
417 | /* |
418 | ** Initialize a WHERE clause scanner object. Return a pointer to the |
419 | ** first match. Return NULL if there are no matches. |
420 | ** |
421 | ** The scanner will be searching the WHERE clause pWC. It will look |
422 | ** for terms of the form "X <op> <expr>" where X is column iColumn of table |
423 | ** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx |
424 | ** must be one of the indexes of table iCur. |
425 | ** |
426 | ** The <op> must be one of the operators described by opMask. |
427 | ** |
428 | ** If the search is for X and the WHERE clause contains terms of the |
429 | ** form X=Y then this routine might also return terms of the form |
430 | ** "Y <op> <expr>". The number of levels of transitivity is limited, |
431 | ** but is enough to handle most commonly occurring SQL statements. |
432 | ** |
433 | ** If X is not the INTEGER PRIMARY KEY then X must be compatible with |
434 | ** index pIdx. |
435 | */ |
436 | static WhereTerm *whereScanInit( |
437 | WhereScan *pScan, /* The WhereScan object being initialized */ |
438 | WhereClause *pWC, /* The WHERE clause to be scanned */ |
439 | int iCur, /* Cursor to scan for */ |
440 | int iColumn, /* Column to scan for */ |
441 | u32 opMask, /* Operator(s) to scan for */ |
442 | Index *pIdx /* Must be compatible with this index */ |
443 | ){ |
444 | pScan->pOrigWC = pWC; |
445 | pScan->pWC = pWC; |
446 | pScan->pIdxExpr = 0; |
447 | pScan->idxaff = 0; |
448 | pScan->zCollName = 0; |
449 | pScan->opMask = opMask; |
450 | pScan->k = 0; |
451 | pScan->aiCur[0] = iCur; |
452 | pScan->nEquiv = 1; |
453 | pScan->iEquiv = 1; |
454 | if( pIdx ){ |
455 | int j = iColumn; |
456 | iColumn = pIdx->aiColumn[j]; |
457 | if( iColumn==pIdx->pTable->iPKey ){ |
458 | iColumn = XN_ROWID; |
459 | }else if( iColumn>=0 ){ |
460 | pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; |
461 | pScan->zCollName = pIdx->azColl[j]; |
462 | }else if( iColumn==XN_EXPR ){ |
463 | pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; |
464 | pScan->zCollName = pIdx->azColl[j]; |
465 | pScan->aiColumn[0] = XN_EXPR; |
466 | return whereScanInitIndexExpr(pScan); |
467 | } |
468 | }else if( iColumn==XN_EXPR ){ |
469 | return 0; |
470 | } |
471 | pScan->aiColumn[0] = iColumn; |
472 | return whereScanNext(pScan); |
473 | } |
474 | |
475 | /* |
476 | ** Search for a term in the WHERE clause that is of the form "X <op> <expr>" |
477 | ** where X is a reference to the iColumn of table iCur or of index pIdx |
478 | ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by |
479 | ** the op parameter. Return a pointer to the term. Return 0 if not found. |
480 | ** |
481 | ** If pIdx!=0 then it must be one of the indexes of table iCur. |
482 | ** Search for terms matching the iColumn-th column of pIdx |
483 | ** rather than the iColumn-th column of table iCur. |
484 | ** |
485 | ** The term returned might by Y=<expr> if there is another constraint in |
486 | ** the WHERE clause that specifies that X=Y. Any such constraints will be |
487 | ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The |
488 | ** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 |
489 | ** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 |
490 | ** other equivalent values. Hence a search for X will return <expr> if X=A1 |
491 | ** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>. |
492 | ** |
493 | ** If there are multiple terms in the WHERE clause of the form "X <op> <expr>" |
494 | ** then try for the one with no dependencies on <expr> - in other words where |
495 | ** <expr> is a constant expression of some kind. Only return entries of |
496 | ** the form "X <op> Y" where Y is a column in another table if no terms of |
497 | ** the form "X <op> <const-expr>" exist. If no terms with a constant RHS |
498 | ** exist, try to return a term that does not use WO_EQUIV. |
499 | */ |
500 | WhereTerm *sqlite3WhereFindTerm( |
501 | WhereClause *pWC, /* The WHERE clause to be searched */ |
502 | int iCur, /* Cursor number of LHS */ |
503 | int iColumn, /* Column number of LHS */ |
504 | Bitmask notReady, /* RHS must not overlap with this mask */ |
505 | u32 op, /* Mask of WO_xx values describing operator */ |
506 | Index *pIdx /* Must be compatible with this index, if not NULL */ |
507 | ){ |
508 | WhereTerm *pResult = 0; |
509 | WhereTerm *p; |
510 | WhereScan scan; |
511 | |
512 | p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); |
513 | op &= WO_EQ|WO_IS; |
514 | while( p ){ |
515 | if( (p->prereqRight & notReady)==0 ){ |
516 | if( p->prereqRight==0 && (p->eOperator&op)!=0 ){ |
517 | testcase( p->eOperator & WO_IS ); |
518 | return p; |
519 | } |
520 | if( pResult==0 ) pResult = p; |
521 | } |
522 | p = whereScanNext(&scan); |
523 | } |
524 | return pResult; |
525 | } |
526 | |
527 | /* |
528 | ** This function searches pList for an entry that matches the iCol-th column |
529 | ** of index pIdx. |
530 | ** |
531 | ** If such an expression is found, its index in pList->a[] is returned. If |
532 | ** no expression is found, -1 is returned. |
533 | */ |
534 | static int findIndexCol( |
535 | Parse *pParse, /* Parse context */ |
536 | ExprList *pList, /* Expression list to search */ |
537 | int iBase, /* Cursor for table associated with pIdx */ |
538 | Index *pIdx, /* Index to match column of */ |
539 | int iCol /* Column of index to match */ |
540 | ){ |
541 | int i; |
542 | const char *zColl = pIdx->azColl[iCol]; |
543 | |
544 | for(i=0; i<pList->nExpr; i++){ |
545 | Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr); |
546 | if( ALWAYS(p!=0) |
547 | && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) |
548 | && p->iColumn==pIdx->aiColumn[iCol] |
549 | && p->iTable==iBase |
550 | ){ |
551 | CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr); |
552 | if( 0==sqlite3StrICmp(pColl->zName, zColl) ){ |
553 | return i; |
554 | } |
555 | } |
556 | } |
557 | |
558 | return -1; |
559 | } |
560 | |
561 | /* |
562 | ** Return TRUE if the iCol-th column of index pIdx is NOT NULL |
563 | */ |
564 | static int indexColumnNotNull(Index *pIdx, int iCol){ |
565 | int j; |
566 | assert( pIdx!=0 ); |
567 | assert( iCol>=0 && iCol<pIdx->nColumn ); |
568 | j = pIdx->aiColumn[iCol]; |
569 | if( j>=0 ){ |
570 | return pIdx->pTable->aCol[j].notNull; |
571 | }else if( j==(-1) ){ |
572 | return 1; |
573 | }else{ |
574 | assert( j==(-2) ); |
575 | return 0; /* Assume an indexed expression can always yield a NULL */ |
576 | |
577 | } |
578 | } |
579 | |
580 | /* |
581 | ** Return true if the DISTINCT expression-list passed as the third argument |
582 | ** is redundant. |
583 | ** |
584 | ** A DISTINCT list is redundant if any subset of the columns in the |
585 | ** DISTINCT list are collectively unique and individually non-null. |
586 | */ |
587 | static int isDistinctRedundant( |
588 | Parse *pParse, /* Parsing context */ |
589 | SrcList *pTabList, /* The FROM clause */ |
590 | WhereClause *pWC, /* The WHERE clause */ |
591 | ExprList *pDistinct /* The result set that needs to be DISTINCT */ |
592 | ){ |
593 | Table *pTab; |
594 | Index *pIdx; |
595 | int i; |
596 | int iBase; |
597 | |
598 | /* If there is more than one table or sub-select in the FROM clause of |
599 | ** this query, then it will not be possible to show that the DISTINCT |
600 | ** clause is redundant. */ |
601 | if( pTabList->nSrc!=1 ) return 0; |
602 | iBase = pTabList->a[0].iCursor; |
603 | pTab = pTabList->a[0].pTab; |
604 | |
605 | /* If any of the expressions is an IPK column on table iBase, then return |
606 | ** true. Note: The (p->iTable==iBase) part of this test may be false if the |
607 | ** current SELECT is a correlated sub-query. |
608 | */ |
609 | for(i=0; i<pDistinct->nExpr; i++){ |
610 | Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr); |
611 | if( NEVER(p==0) ) continue; |
612 | if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue; |
613 | if( p->iTable==iBase && p->iColumn<0 ) return 1; |
614 | } |
615 | |
616 | /* Loop through all indices on the table, checking each to see if it makes |
617 | ** the DISTINCT qualifier redundant. It does so if: |
618 | ** |
619 | ** 1. The index is itself UNIQUE, and |
620 | ** |
621 | ** 2. All of the columns in the index are either part of the pDistinct |
622 | ** list, or else the WHERE clause contains a term of the form "col=X", |
623 | ** where X is a constant value. The collation sequences of the |
624 | ** comparison and select-list expressions must match those of the index. |
625 | ** |
626 | ** 3. All of those index columns for which the WHERE clause does not |
627 | ** contain a "col=X" term are subject to a NOT NULL constraint. |
628 | */ |
629 | for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ |
630 | if( !IsUniqueIndex(pIdx) ) continue; |
631 | if( pIdx->pPartIdxWhere ) continue; |
632 | for(i=0; i<pIdx->nKeyCol; i++){ |
633 | if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ |
634 | if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; |
635 | if( indexColumnNotNull(pIdx, i)==0 ) break; |
636 | } |
637 | } |
638 | if( i==pIdx->nKeyCol ){ |
639 | /* This index implies that the DISTINCT qualifier is redundant. */ |
640 | return 1; |
641 | } |
642 | } |
643 | |
644 | return 0; |
645 | } |
646 | |
647 | |
648 | /* |
649 | ** Estimate the logarithm of the input value to base 2. |
650 | */ |
651 | static LogEst estLog(LogEst N){ |
652 | return N<=10 ? 0 : sqlite3LogEst(N) - 33; |
653 | } |
654 | |
655 | /* |
656 | ** Convert OP_Column opcodes to OP_Copy in previously generated code. |
657 | ** |
658 | ** This routine runs over generated VDBE code and translates OP_Column |
659 | ** opcodes into OP_Copy when the table is being accessed via co-routine |
660 | ** instead of via table lookup. |
661 | ** |
662 | ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on |
663 | ** cursor iTabCur are transformed into OP_Sequence opcode for the |
664 | ** iAutoidxCur cursor, in order to generate unique rowids for the |
665 | ** automatic index being generated. |
666 | */ |
667 | static void translateColumnToCopy( |
668 | Parse *pParse, /* Parsing context */ |
669 | int iStart, /* Translate from this opcode to the end */ |
670 | int iTabCur, /* OP_Column/OP_Rowid references to this table */ |
671 | int iRegister, /* The first column is in this register */ |
672 | int iAutoidxCur /* If non-zero, cursor of autoindex being generated */ |
673 | ){ |
674 | Vdbe *v = pParse->pVdbe; |
675 | VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); |
676 | int iEnd = sqlite3VdbeCurrentAddr(v); |
677 | if( pParse->db->mallocFailed ) return; |
678 | for(; iStart<iEnd; iStart++, pOp++){ |
679 | if( pOp->p1!=iTabCur ) continue; |
680 | if( pOp->opcode==OP_Column ){ |
681 | pOp->opcode = OP_Copy; |
682 | pOp->p1 = pOp->p2 + iRegister; |
683 | pOp->p2 = pOp->p3; |
684 | pOp->p3 = 0; |
685 | pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */ |
686 | }else if( pOp->opcode==OP_Rowid ){ |
687 | pOp->opcode = OP_Sequence; |
688 | pOp->p1 = iAutoidxCur; |
689 | #ifdef SQLITE_ALLOW_ROWID_IN_VIEW |
690 | if( iAutoidxCur==0 ){ |
691 | pOp->opcode = OP_Null; |
692 | pOp->p3 = 0; |
693 | } |
694 | #endif |
695 | } |
696 | } |
697 | } |
698 | |
699 | /* |
700 | ** Two routines for printing the content of an sqlite3_index_info |
701 | ** structure. Used for testing and debugging only. If neither |
702 | ** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines |
703 | ** are no-ops. |
704 | */ |
705 | #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) |
706 | static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ |
707 | int i; |
708 | if( !sqlite3WhereTrace ) return; |
709 | for(i=0; i<p->nConstraint; i++){ |
710 | sqlite3DebugPrintf( |
711 | " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n" , |
712 | i, |
713 | p->aConstraint[i].iColumn, |
714 | p->aConstraint[i].iTermOffset, |
715 | p->aConstraint[i].op, |
716 | p->aConstraint[i].usable, |
717 | sqlite3_vtab_collation(p,i)); |
718 | } |
719 | for(i=0; i<p->nOrderBy; i++){ |
720 | sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n" , |
721 | i, |
722 | p->aOrderBy[i].iColumn, |
723 | p->aOrderBy[i].desc); |
724 | } |
725 | } |
726 | static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ |
727 | int i; |
728 | if( !sqlite3WhereTrace ) return; |
729 | for(i=0; i<p->nConstraint; i++){ |
730 | sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n" , |
731 | i, |
732 | p->aConstraintUsage[i].argvIndex, |
733 | p->aConstraintUsage[i].omit); |
734 | } |
735 | sqlite3DebugPrintf(" idxNum=%d\n" , p->idxNum); |
736 | sqlite3DebugPrintf(" idxStr=%s\n" , p->idxStr); |
737 | sqlite3DebugPrintf(" orderByConsumed=%d\n" , p->orderByConsumed); |
738 | sqlite3DebugPrintf(" estimatedCost=%g\n" , p->estimatedCost); |
739 | sqlite3DebugPrintf(" estimatedRows=%lld\n" , p->estimatedRows); |
740 | } |
741 | #else |
742 | #define whereTraceIndexInfoInputs(A) |
743 | #define whereTraceIndexInfoOutputs(A) |
744 | #endif |
745 | |
746 | /* |
747 | ** We know that pSrc is an operand of an outer join. Return true if |
748 | ** pTerm is a constraint that is compatible with that join. |
749 | ** |
750 | ** pTerm must be EP_OuterON if pSrc is the right operand of an |
751 | ** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc |
752 | ** is the left operand of a RIGHT join. |
753 | ** |
754 | ** See https://sqlite.org/forum/forumpost/206d99a16dd9212f |
755 | ** for an example of a WHERE clause constraints that may not be used on |
756 | ** the right table of a RIGHT JOIN because the constraint implies a |
757 | ** not-NULL condition on the left table of the RIGHT JOIN. |
758 | */ |
759 | static int constraintCompatibleWithOuterJoin( |
760 | const WhereTerm *pTerm, /* WHERE clause term to check */ |
761 | const SrcItem *pSrc /* Table we are trying to access */ |
762 | ){ |
763 | assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */ |
764 | testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); |
765 | testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); |
766 | testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) |
767 | testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); |
768 | if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) |
769 | || pTerm->pExpr->w.iJoin != pSrc->iCursor |
770 | ){ |
771 | return 0; |
772 | } |
773 | if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0 |
774 | && ExprHasProperty(pTerm->pExpr, EP_InnerON) |
775 | ){ |
776 | return 0; |
777 | } |
778 | return 1; |
779 | } |
780 | |
781 | |
782 | |
783 | #ifndef SQLITE_OMIT_AUTOMATIC_INDEX |
784 | /* |
785 | ** Return TRUE if the WHERE clause term pTerm is of a form where it |
786 | ** could be used with an index to access pSrc, assuming an appropriate |
787 | ** index existed. |
788 | */ |
789 | static int termCanDriveIndex( |
790 | const WhereTerm *pTerm, /* WHERE clause term to check */ |
791 | const SrcItem *pSrc, /* Table we are trying to access */ |
792 | const Bitmask notReady /* Tables in outer loops of the join */ |
793 | ){ |
794 | char aff; |
795 | if( pTerm->leftCursor!=pSrc->iCursor ) return 0; |
796 | if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; |
797 | assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); |
798 | if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 |
799 | && !constraintCompatibleWithOuterJoin(pTerm,pSrc) |
800 | ){ |
801 | return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */ |
802 | } |
803 | if( (pTerm->prereqRight & notReady)!=0 ) return 0; |
804 | assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); |
805 | if( pTerm->u.x.leftColumn<0 ) return 0; |
806 | aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity; |
807 | if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; |
808 | testcase( pTerm->pExpr->op==TK_IS ); |
809 | return 1; |
810 | } |
811 | #endif |
812 | |
813 | |
814 | #ifndef SQLITE_OMIT_AUTOMATIC_INDEX |
815 | /* |
816 | ** Generate code to construct the Index object for an automatic index |
817 | ** and to set up the WhereLevel object pLevel so that the code generator |
818 | ** makes use of the automatic index. |
819 | */ |
820 | static SQLITE_NOINLINE void constructAutomaticIndex( |
821 | Parse *pParse, /* The parsing context */ |
822 | const WhereClause *pWC, /* The WHERE clause */ |
823 | const SrcItem *pSrc, /* The FROM clause term to get the next index */ |
824 | const Bitmask notReady, /* Mask of cursors that are not available */ |
825 | WhereLevel *pLevel /* Write new index here */ |
826 | ){ |
827 | int nKeyCol; /* Number of columns in the constructed index */ |
828 | WhereTerm *pTerm; /* A single term of the WHERE clause */ |
829 | WhereTerm *pWCEnd; /* End of pWC->a[] */ |
830 | Index *pIdx; /* Object describing the transient index */ |
831 | Vdbe *v; /* Prepared statement under construction */ |
832 | int addrInit; /* Address of the initialization bypass jump */ |
833 | Table *pTable; /* The table being indexed */ |
834 | int addrTop; /* Top of the index fill loop */ |
835 | int regRecord; /* Register holding an index record */ |
836 | int n; /* Column counter */ |
837 | int i; /* Loop counter */ |
838 | int mxBitCol; /* Maximum column in pSrc->colUsed */ |
839 | CollSeq *pColl; /* Collating sequence to on a column */ |
840 | WhereLoop *pLoop; /* The Loop object */ |
841 | char *zNotUsed; /* Extra space on the end of pIdx */ |
842 | Bitmask idxCols; /* Bitmap of columns used for indexing */ |
843 | Bitmask ; /* Bitmap of additional columns */ |
844 | u8 sentWarning = 0; /* True if a warnning has been issued */ |
845 | Expr *pPartial = 0; /* Partial Index Expression */ |
846 | int iContinue = 0; /* Jump here to skip excluded rows */ |
847 | SrcItem *pTabItem; /* FROM clause term being indexed */ |
848 | int addrCounter = 0; /* Address where integer counter is initialized */ |
849 | int regBase; /* Array of registers where record is assembled */ |
850 | |
851 | /* Generate code to skip over the creation and initialization of the |
852 | ** transient index on 2nd and subsequent iterations of the loop. */ |
853 | v = pParse->pVdbe; |
854 | assert( v!=0 ); |
855 | addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); |
856 | |
857 | /* Count the number of columns that will be added to the index |
858 | ** and used to match WHERE clause constraints */ |
859 | nKeyCol = 0; |
860 | pTable = pSrc->pTab; |
861 | pWCEnd = &pWC->a[pWC->nTerm]; |
862 | pLoop = pLevel->pWLoop; |
863 | idxCols = 0; |
864 | for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){ |
865 | Expr *pExpr = pTerm->pExpr; |
866 | /* Make the automatic index a partial index if there are terms in the |
867 | ** WHERE clause (or the ON clause of a LEFT join) that constrain which |
868 | ** rows of the target table (pSrc) that can be used. */ |
869 | if( (pTerm->wtFlags & TERM_VIRTUAL)==0 |
870 | && sqlite3ExprIsTableConstraint(pExpr, pSrc) |
871 | ){ |
872 | pPartial = sqlite3ExprAnd(pParse, pPartial, |
873 | sqlite3ExprDup(pParse->db, pExpr, 0)); |
874 | } |
875 | if( termCanDriveIndex(pTerm, pSrc, notReady) ){ |
876 | int iCol; |
877 | Bitmask cMask; |
878 | assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); |
879 | iCol = pTerm->u.x.leftColumn; |
880 | cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); |
881 | testcase( iCol==BMS ); |
882 | testcase( iCol==BMS-1 ); |
883 | if( !sentWarning ){ |
884 | sqlite3_log(SQLITE_WARNING_AUTOINDEX, |
885 | "automatic index on %s(%s)" , pTable->zName, |
886 | pTable->aCol[iCol].zCnName); |
887 | sentWarning = 1; |
888 | } |
889 | if( (idxCols & cMask)==0 ){ |
890 | if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ |
891 | goto end_auto_index_create; |
892 | } |
893 | pLoop->aLTerm[nKeyCol++] = pTerm; |
894 | idxCols |= cMask; |
895 | } |
896 | } |
897 | } |
898 | assert( nKeyCol>0 || pParse->db->mallocFailed ); |
899 | pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; |
900 | pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED |
901 | | WHERE_AUTO_INDEX; |
902 | |
903 | /* Count the number of additional columns needed to create a |
904 | ** covering index. A "covering index" is an index that contains all |
905 | ** columns that are needed by the query. With a covering index, the |
906 | ** original table never needs to be accessed. Automatic indices must |
907 | ** be a covering index because the index will not be updated if the |
908 | ** original table changes and the index and table cannot both be used |
909 | ** if they go out of sync. |
910 | */ |
911 | extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); |
912 | mxBitCol = MIN(BMS-1,pTable->nCol); |
913 | testcase( pTable->nCol==BMS-1 ); |
914 | testcase( pTable->nCol==BMS-2 ); |
915 | for(i=0; i<mxBitCol; i++){ |
916 | if( extraCols & MASKBIT(i) ) nKeyCol++; |
917 | } |
918 | if( pSrc->colUsed & MASKBIT(BMS-1) ){ |
919 | nKeyCol += pTable->nCol - BMS + 1; |
920 | } |
921 | |
922 | /* Construct the Index object to describe this index */ |
923 | pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); |
924 | if( pIdx==0 ) goto end_auto_index_create; |
925 | pLoop->u.btree.pIndex = pIdx; |
926 | pIdx->zName = "auto-index" ; |
927 | pIdx->pTable = pTable; |
928 | n = 0; |
929 | idxCols = 0; |
930 | for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){ |
931 | if( termCanDriveIndex(pTerm, pSrc, notReady) ){ |
932 | int iCol; |
933 | Bitmask cMask; |
934 | assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); |
935 | iCol = pTerm->u.x.leftColumn; |
936 | cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); |
937 | testcase( iCol==BMS-1 ); |
938 | testcase( iCol==BMS ); |
939 | if( (idxCols & cMask)==0 ){ |
940 | Expr *pX = pTerm->pExpr; |
941 | idxCols |= cMask; |
942 | pIdx->aiColumn[n] = pTerm->u.x.leftColumn; |
943 | pColl = sqlite3ExprCompareCollSeq(pParse, pX); |
944 | assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ |
945 | pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; |
946 | n++; |
947 | } |
948 | } |
949 | } |
950 | assert( (u32)n==pLoop->u.btree.nEq ); |
951 | |
952 | /* Add additional columns needed to make the automatic index into |
953 | ** a covering index */ |
954 | for(i=0; i<mxBitCol; i++){ |
955 | if( extraCols & MASKBIT(i) ){ |
956 | pIdx->aiColumn[n] = i; |
957 | pIdx->azColl[n] = sqlite3StrBINARY; |
958 | n++; |
959 | } |
960 | } |
961 | if( pSrc->colUsed & MASKBIT(BMS-1) ){ |
962 | for(i=BMS-1; i<pTable->nCol; i++){ |
963 | pIdx->aiColumn[n] = i; |
964 | pIdx->azColl[n] = sqlite3StrBINARY; |
965 | n++; |
966 | } |
967 | } |
968 | assert( n==nKeyCol ); |
969 | pIdx->aiColumn[n] = XN_ROWID; |
970 | pIdx->azColl[n] = sqlite3StrBINARY; |
971 | |
972 | /* Create the automatic index */ |
973 | assert( pLevel->iIdxCur>=0 ); |
974 | pLevel->iIdxCur = pParse->nTab++; |
975 | sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); |
976 | sqlite3VdbeSetP4KeyInfo(pParse, pIdx); |
977 | VdbeComment((v, "for %s" , pTable->zName)); |
978 | if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ |
979 | pLevel->regFilter = ++pParse->nMem; |
980 | sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); |
981 | } |
982 | |
983 | /* Fill the automatic index with content */ |
984 | pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; |
985 | if( pTabItem->fg.viaCoroutine ){ |
986 | int regYield = pTabItem->regReturn; |
987 | addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); |
988 | sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); |
989 | addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); |
990 | VdbeCoverage(v); |
991 | VdbeComment((v, "next row of %s" , pTabItem->pTab->zName)); |
992 | }else{ |
993 | addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); |
994 | } |
995 | if( pPartial ){ |
996 | iContinue = sqlite3VdbeMakeLabel(pParse); |
997 | sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); |
998 | pLoop->wsFlags |= WHERE_PARTIALIDX; |
999 | } |
1000 | regRecord = sqlite3GetTempReg(pParse); |
1001 | regBase = sqlite3GenerateIndexKey( |
1002 | pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 |
1003 | ); |
1004 | if( pLevel->regFilter ){ |
1005 | sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, |
1006 | regBase, pLoop->u.btree.nEq); |
1007 | } |
1008 | sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); |
1009 | sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); |
1010 | if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); |
1011 | if( pTabItem->fg.viaCoroutine ){ |
1012 | sqlite3VdbeChangeP2(v, addrCounter, regBase+n); |
1013 | testcase( pParse->db->mallocFailed ); |
1014 | assert( pLevel->iIdxCur>0 ); |
1015 | translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, |
1016 | pTabItem->regResult, pLevel->iIdxCur); |
1017 | sqlite3VdbeGoto(v, addrTop); |
1018 | pTabItem->fg.viaCoroutine = 0; |
1019 | }else{ |
1020 | sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); |
1021 | sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); |
1022 | } |
1023 | sqlite3VdbeJumpHere(v, addrTop); |
1024 | sqlite3ReleaseTempReg(pParse, regRecord); |
1025 | |
1026 | /* Jump here when skipping the initialization */ |
1027 | sqlite3VdbeJumpHere(v, addrInit); |
1028 | |
1029 | end_auto_index_create: |
1030 | sqlite3ExprDelete(pParse->db, pPartial); |
1031 | } |
1032 | #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ |
1033 | |
1034 | /* |
1035 | ** Generate bytecode that will initialize a Bloom filter that is appropriate |
1036 | ** for pLevel. |
1037 | ** |
1038 | ** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER |
1039 | ** flag set, initialize a Bloomfilter for them as well. Except don't do |
1040 | ** this recursive initialization if the SQLITE_BloomPulldown optimization has |
1041 | ** been turned off. |
1042 | ** |
1043 | ** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared |
1044 | ** from the loop, but the regFilter value is set to a register that implements |
1045 | ** the Bloom filter. When regFilter is positive, the |
1046 | ** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter |
1047 | ** and skip the subsequence B-Tree seek if the Bloom filter indicates that |
1048 | ** no matching rows exist. |
1049 | ** |
1050 | ** This routine may only be called if it has previously been determined that |
1051 | ** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit |
1052 | ** is set. |
1053 | */ |
1054 | static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( |
1055 | WhereInfo *pWInfo, /* The WHERE clause */ |
1056 | int iLevel, /* Index in pWInfo->a[] that is pLevel */ |
1057 | WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */ |
1058 | Bitmask notReady /* Loops that are not ready */ |
1059 | ){ |
1060 | int addrOnce; /* Address of opening OP_Once */ |
1061 | int addrTop; /* Address of OP_Rewind */ |
1062 | int addrCont; /* Jump here to skip a row */ |
1063 | const WhereTerm *pTerm; /* For looping over WHERE clause terms */ |
1064 | const WhereTerm *pWCEnd; /* Last WHERE clause term */ |
1065 | Parse *pParse = pWInfo->pParse; /* Parsing context */ |
1066 | Vdbe *v = pParse->pVdbe; /* VDBE under construction */ |
1067 | WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ |
1068 | int iCur; /* Cursor for table getting the filter */ |
1069 | |
1070 | assert( pLoop!=0 ); |
1071 | assert( v!=0 ); |
1072 | assert( pLoop->wsFlags & WHERE_BLOOMFILTER ); |
1073 | |
1074 | addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); |
1075 | do{ |
1076 | const SrcItem *pItem; |
1077 | const Table *pTab; |
1078 | u64 sz; |
1079 | sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel); |
1080 | addrCont = sqlite3VdbeMakeLabel(pParse); |
1081 | iCur = pLevel->iTabCur; |
1082 | pLevel->regFilter = ++pParse->nMem; |
1083 | |
1084 | /* The Bloom filter is a Blob held in a register. Initialize it |
1085 | ** to zero-filled blob of at least 80K bits, but maybe more if the |
1086 | ** estimated size of the table is larger. We could actually |
1087 | ** measure the size of the table at run-time using OP_Count with |
1088 | ** P3==1 and use that value to initialize the blob. But that makes |
1089 | ** testing complicated. By basing the blob size on the value in the |
1090 | ** sqlite_stat1 table, testing is much easier. |
1091 | */ |
1092 | pItem = &pWInfo->pTabList->a[pLevel->iFrom]; |
1093 | assert( pItem!=0 ); |
1094 | pTab = pItem->pTab; |
1095 | assert( pTab!=0 ); |
1096 | sz = sqlite3LogEstToInt(pTab->nRowLogEst); |
1097 | if( sz<10000 ){ |
1098 | sz = 10000; |
1099 | }else if( sz>10000000 ){ |
1100 | sz = 10000000; |
1101 | } |
1102 | sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter); |
1103 | |
1104 | addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); |
1105 | pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm]; |
1106 | for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){ |
1107 | Expr *pExpr = pTerm->pExpr; |
1108 | if( (pTerm->wtFlags & TERM_VIRTUAL)==0 |
1109 | && sqlite3ExprIsTableConstraint(pExpr, pItem) |
1110 | ){ |
1111 | sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); |
1112 | } |
1113 | } |
1114 | if( pLoop->wsFlags & WHERE_IPK ){ |
1115 | int r1 = sqlite3GetTempReg(pParse); |
1116 | sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1); |
1117 | sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1); |
1118 | sqlite3ReleaseTempReg(pParse, r1); |
1119 | }else{ |
1120 | Index *pIdx = pLoop->u.btree.pIndex; |
1121 | int n = pLoop->u.btree.nEq; |
1122 | int r1 = sqlite3GetTempRange(pParse, n); |
1123 | int jj; |
1124 | for(jj=0; jj<n; jj++){ |
1125 | int iCol = pIdx->aiColumn[jj]; |
1126 | assert( pIdx->pTable==pItem->pTab ); |
1127 | sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj); |
1128 | } |
1129 | sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); |
1130 | sqlite3ReleaseTempRange(pParse, r1, n); |
1131 | } |
1132 | sqlite3VdbeResolveLabel(v, addrCont); |
1133 | sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); |
1134 | VdbeCoverage(v); |
1135 | sqlite3VdbeJumpHere(v, addrTop); |
1136 | pLoop->wsFlags &= ~WHERE_BLOOMFILTER; |
1137 | if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break; |
1138 | while( ++iLevel < pWInfo->nLevel ){ |
1139 | const SrcItem *pTabItem; |
1140 | pLevel = &pWInfo->a[iLevel]; |
1141 | pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; |
1142 | if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue; |
1143 | pLoop = pLevel->pWLoop; |
1144 | if( NEVER(pLoop==0) ) continue; |
1145 | if( pLoop->prereq & notReady ) continue; |
1146 | if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN)) |
1147 | ==WHERE_BLOOMFILTER |
1148 | ){ |
1149 | /* This is a candidate for bloom-filter pull-down (early evaluation). |
1150 | ** The test that WHERE_COLUMN_IN is omitted is important, as we are |
1151 | ** not able to do early evaluation of bloom filters that make use of |
1152 | ** the IN operator */ |
1153 | break; |
1154 | } |
1155 | } |
1156 | }while( iLevel < pWInfo->nLevel ); |
1157 | sqlite3VdbeJumpHere(v, addrOnce); |
1158 | } |
1159 | |
1160 | |
1161 | #ifndef SQLITE_OMIT_VIRTUALTABLE |
1162 | /* |
1163 | ** Allocate and populate an sqlite3_index_info structure. It is the |
1164 | ** responsibility of the caller to eventually release the structure |
1165 | ** by passing the pointer returned by this function to freeIndexInfo(). |
1166 | */ |
1167 | static sqlite3_index_info *allocateIndexInfo( |
1168 | WhereInfo *pWInfo, /* The WHERE clause */ |
1169 | WhereClause *pWC, /* The WHERE clause being analyzed */ |
1170 | Bitmask mUnusable, /* Ignore terms with these prereqs */ |
1171 | SrcItem *pSrc, /* The FROM clause term that is the vtab */ |
1172 | u16 *pmNoOmit /* Mask of terms not to omit */ |
1173 | ){ |
1174 | int i, j; |
1175 | int nTerm; |
1176 | Parse *pParse = pWInfo->pParse; |
1177 | struct sqlite3_index_constraint *pIdxCons; |
1178 | struct sqlite3_index_orderby *pIdxOrderBy; |
1179 | struct sqlite3_index_constraint_usage *pUsage; |
1180 | struct HiddenIndexInfo *pHidden; |
1181 | WhereTerm *pTerm; |
1182 | int nOrderBy; |
1183 | sqlite3_index_info *pIdxInfo; |
1184 | u16 mNoOmit = 0; |
1185 | const Table *pTab; |
1186 | int eDistinct = 0; |
1187 | ExprList *pOrderBy = pWInfo->pOrderBy; |
1188 | |
1189 | assert( pSrc!=0 ); |
1190 | pTab = pSrc->pTab; |
1191 | assert( pTab!=0 ); |
1192 | assert( IsVirtual(pTab) ); |
1193 | |
1194 | /* Find all WHERE clause constraints referring to this virtual table. |
1195 | ** Mark each term with the TERM_OK flag. Set nTerm to the number of |
1196 | ** terms found. |
1197 | */ |
1198 | for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ |
1199 | pTerm->wtFlags &= ~TERM_OK; |
1200 | if( pTerm->leftCursor != pSrc->iCursor ) continue; |
1201 | if( pTerm->prereqRight & mUnusable ) continue; |
1202 | assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); |
1203 | testcase( pTerm->eOperator & WO_IN ); |
1204 | testcase( pTerm->eOperator & WO_ISNULL ); |
1205 | testcase( pTerm->eOperator & WO_IS ); |
1206 | testcase( pTerm->eOperator & WO_ALL ); |
1207 | if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; |
1208 | if( pTerm->wtFlags & TERM_VNULL ) continue; |
1209 | |
1210 | assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); |
1211 | assert( pTerm->u.x.leftColumn>=XN_ROWID ); |
1212 | assert( pTerm->u.x.leftColumn<pTab->nCol ); |
1213 | if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 |
1214 | && !constraintCompatibleWithOuterJoin(pTerm,pSrc) |
1215 | ){ |
1216 | continue; |
1217 | } |
1218 | nTerm++; |
1219 | pTerm->wtFlags |= TERM_OK; |
1220 | } |
1221 | |
1222 | /* If the ORDER BY clause contains only columns in the current |
1223 | ** virtual table then allocate space for the aOrderBy part of |
1224 | ** the sqlite3_index_info structure. |
1225 | */ |
1226 | nOrderBy = 0; |
1227 | if( pOrderBy ){ |
1228 | int n = pOrderBy->nExpr; |
1229 | for(i=0; i<n; i++){ |
1230 | Expr *pExpr = pOrderBy->a[i].pExpr; |
1231 | Expr *pE2; |
1232 | |
1233 | /* Skip over constant terms in the ORDER BY clause */ |
1234 | if( sqlite3ExprIsConstant(pExpr) ){ |
1235 | continue; |
1236 | } |
1237 | |
1238 | /* Virtual tables are unable to deal with NULLS FIRST */ |
1239 | if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break; |
1240 | |
1241 | /* First case - a direct column references without a COLLATE operator */ |
1242 | if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){ |
1243 | assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumn<pTab->nCol ); |
1244 | continue; |
1245 | } |
1246 | |
1247 | /* 2nd case - a column reference with a COLLATE operator. Only match |
1248 | ** of the COLLATE operator matches the collation of the column. */ |
1249 | if( pExpr->op==TK_COLLATE |
1250 | && (pE2 = pExpr->pLeft)->op==TK_COLUMN |
1251 | && pE2->iTable==pSrc->iCursor |
1252 | ){ |
1253 | const char *zColl; /* The collating sequence name */ |
1254 | assert( !ExprHasProperty(pExpr, EP_IntValue) ); |
1255 | assert( pExpr->u.zToken!=0 ); |
1256 | assert( pE2->iColumn>=XN_ROWID && pE2->iColumn<pTab->nCol ); |
1257 | pExpr->iColumn = pE2->iColumn; |
1258 | if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */ |
1259 | zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]); |
1260 | if( zColl==0 ) zColl = sqlite3StrBINARY; |
1261 | if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue; |
1262 | } |
1263 | |
1264 | /* No matches cause a break out of the loop */ |
1265 | break; |
1266 | } |
1267 | if( i==n ){ |
1268 | nOrderBy = n; |
1269 | if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ |
1270 | eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); |
1271 | }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ |
1272 | eDistinct = 1; |
1273 | } |
1274 | } |
1275 | } |
1276 | |
1277 | /* Allocate the sqlite3_index_info structure |
1278 | */ |
1279 | pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) |
1280 | + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm |
1281 | + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) |
1282 | + sizeof(sqlite3_value*)*nTerm ); |
1283 | if( pIdxInfo==0 ){ |
1284 | sqlite3ErrorMsg(pParse, "out of memory" ); |
1285 | return 0; |
1286 | } |
1287 | pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1]; |
1288 | pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm]; |
1289 | pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; |
1290 | pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; |
1291 | pIdxInfo->aConstraint = pIdxCons; |
1292 | pIdxInfo->aOrderBy = pIdxOrderBy; |
1293 | pIdxInfo->aConstraintUsage = pUsage; |
1294 | pHidden->pWC = pWC; |
1295 | pHidden->pParse = pParse; |
1296 | pHidden->eDistinct = eDistinct; |
1297 | pHidden->mIn = 0; |
1298 | for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ |
1299 | u16 op; |
1300 | if( (pTerm->wtFlags & TERM_OK)==0 ) continue; |
1301 | pIdxCons[j].iColumn = pTerm->u.x.leftColumn; |
1302 | pIdxCons[j].iTermOffset = i; |
1303 | op = pTerm->eOperator & WO_ALL; |
1304 | if( op==WO_IN ){ |
1305 | if( (pTerm->wtFlags & TERM_SLICE)==0 ){ |
1306 | pHidden->mIn |= SMASKBIT32(j); |
1307 | } |
1308 | op = WO_EQ; |
1309 | } |
1310 | if( op==WO_AUX ){ |
1311 | pIdxCons[j].op = pTerm->eMatchOp; |
1312 | }else if( op & (WO_ISNULL|WO_IS) ){ |
1313 | if( op==WO_ISNULL ){ |
1314 | pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL; |
1315 | }else{ |
1316 | pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS; |
1317 | } |
1318 | }else{ |
1319 | pIdxCons[j].op = (u8)op; |
1320 | /* The direct assignment in the previous line is possible only because |
1321 | ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The |
1322 | ** following asserts verify this fact. */ |
1323 | assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); |
1324 | assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); |
1325 | assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); |
1326 | assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); |
1327 | assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); |
1328 | assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) ); |
1329 | |
1330 | if( op & (WO_LT|WO_LE|WO_GT|WO_GE) |
1331 | && sqlite3ExprIsVector(pTerm->pExpr->pRight) |
1332 | ){ |
1333 | testcase( j!=i ); |
1334 | if( j<16 ) mNoOmit |= (1 << j); |
1335 | if( op==WO_LT ) pIdxCons[j].op = WO_LE; |
1336 | if( op==WO_GT ) pIdxCons[j].op = WO_GE; |
1337 | } |
1338 | } |
1339 | |
1340 | j++; |
1341 | } |
1342 | assert( j==nTerm ); |
1343 | pIdxInfo->nConstraint = j; |
1344 | for(i=j=0; i<nOrderBy; i++){ |
1345 | Expr *pExpr = pOrderBy->a[i].pExpr; |
1346 | if( sqlite3ExprIsConstant(pExpr) ) continue; |
1347 | assert( pExpr->op==TK_COLUMN |
1348 | || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN |
1349 | && pExpr->iColumn==pExpr->pLeft->iColumn) ); |
1350 | pIdxOrderBy[j].iColumn = pExpr->iColumn; |
1351 | pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC; |
1352 | j++; |
1353 | } |
1354 | pIdxInfo->nOrderBy = j; |
1355 | |
1356 | *pmNoOmit = mNoOmit; |
1357 | return pIdxInfo; |
1358 | } |
1359 | |
1360 | /* |
1361 | ** Free an sqlite3_index_info structure allocated by allocateIndexInfo() |
1362 | ** and possibly modified by xBestIndex methods. |
1363 | */ |
1364 | static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){ |
1365 | HiddenIndexInfo *pHidden; |
1366 | int i; |
1367 | assert( pIdxInfo!=0 ); |
1368 | pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; |
1369 | assert( pHidden->pParse!=0 ); |
1370 | assert( pHidden->pParse->db==db ); |
1371 | for(i=0; i<pIdxInfo->nConstraint; i++){ |
1372 | sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */ |
1373 | pHidden->aRhs[i] = 0; |
1374 | } |
1375 | sqlite3DbFree(db, pIdxInfo); |
1376 | } |
1377 | |
1378 | /* |
1379 | ** The table object reference passed as the second argument to this function |
1380 | ** must represent a virtual table. This function invokes the xBestIndex() |
1381 | ** method of the virtual table with the sqlite3_index_info object that |
1382 | ** comes in as the 3rd argument to this function. |
1383 | ** |
1384 | ** If an error occurs, pParse is populated with an error message and an |
1385 | ** appropriate error code is returned. A return of SQLITE_CONSTRAINT from |
1386 | ** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that |
1387 | ** the current configuration of "unusable" flags in sqlite3_index_info can |
1388 | ** not result in a valid plan. |
1389 | ** |
1390 | ** Whether or not an error is returned, it is the responsibility of the |
1391 | ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates |
1392 | ** that this is required. |
1393 | */ |
1394 | static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ |
1395 | sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; |
1396 | int rc; |
1397 | |
1398 | whereTraceIndexInfoInputs(p); |
1399 | pParse->db->nSchemaLock++; |
1400 | rc = pVtab->pModule->xBestIndex(pVtab, p); |
1401 | pParse->db->nSchemaLock--; |
1402 | whereTraceIndexInfoOutputs(p); |
1403 | |
1404 | if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ |
1405 | if( rc==SQLITE_NOMEM ){ |
1406 | sqlite3OomFault(pParse->db); |
1407 | }else if( !pVtab->zErrMsg ){ |
1408 | sqlite3ErrorMsg(pParse, "%s" , sqlite3ErrStr(rc)); |
1409 | }else{ |
1410 | sqlite3ErrorMsg(pParse, "%s" , pVtab->zErrMsg); |
1411 | } |
1412 | } |
1413 | sqlite3_free(pVtab->zErrMsg); |
1414 | pVtab->zErrMsg = 0; |
1415 | return rc; |
1416 | } |
1417 | #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ |
1418 | |
1419 | #ifdef SQLITE_ENABLE_STAT4 |
1420 | /* |
1421 | ** Estimate the location of a particular key among all keys in an |
1422 | ** index. Store the results in aStat as follows: |
1423 | ** |
1424 | ** aStat[0] Est. number of rows less than pRec |
1425 | ** aStat[1] Est. number of rows equal to pRec |
1426 | ** |
1427 | ** Return the index of the sample that is the smallest sample that |
1428 | ** is greater than or equal to pRec. Note that this index is not an index |
1429 | ** into the aSample[] array - it is an index into a virtual set of samples |
1430 | ** based on the contents of aSample[] and the number of fields in record |
1431 | ** pRec. |
1432 | */ |
1433 | static int whereKeyStats( |
1434 | Parse *pParse, /* Database connection */ |
1435 | Index *pIdx, /* Index to consider domain of */ |
1436 | UnpackedRecord *pRec, /* Vector of values to consider */ |
1437 | int roundUp, /* Round up if true. Round down if false */ |
1438 | tRowcnt *aStat /* OUT: stats written here */ |
1439 | ){ |
1440 | IndexSample *aSample = pIdx->aSample; |
1441 | int iCol; /* Index of required stats in anEq[] etc. */ |
1442 | int i; /* Index of first sample >= pRec */ |
1443 | int iSample; /* Smallest sample larger than or equal to pRec */ |
1444 | int iMin = 0; /* Smallest sample not yet tested */ |
1445 | int iTest; /* Next sample to test */ |
1446 | int res; /* Result of comparison operation */ |
1447 | int nField; /* Number of fields in pRec */ |
1448 | tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ |
1449 | |
1450 | #ifndef SQLITE_DEBUG |
1451 | UNUSED_PARAMETER( pParse ); |
1452 | #endif |
1453 | assert( pRec!=0 ); |
1454 | assert( pIdx->nSample>0 ); |
1455 | assert( pRec->nField>0 ); |
1456 | |
1457 | /* Do a binary search to find the first sample greater than or equal |
1458 | ** to pRec. If pRec contains a single field, the set of samples to search |
1459 | ** is simply the aSample[] array. If the samples in aSample[] contain more |
1460 | ** than one fields, all fields following the first are ignored. |
1461 | ** |
1462 | ** If pRec contains N fields, where N is more than one, then as well as the |
1463 | ** samples in aSample[] (truncated to N fields), the search also has to |
1464 | ** consider prefixes of those samples. For example, if the set of samples |
1465 | ** in aSample is: |
1466 | ** |
1467 | ** aSample[0] = (a, 5) |
1468 | ** aSample[1] = (a, 10) |
1469 | ** aSample[2] = (b, 5) |
1470 | ** aSample[3] = (c, 100) |
1471 | ** aSample[4] = (c, 105) |
1472 | ** |
1473 | ** Then the search space should ideally be the samples above and the |
1474 | ** unique prefixes [a], [b] and [c]. But since that is hard to organize, |
1475 | ** the code actually searches this set: |
1476 | ** |
1477 | ** 0: (a) |
1478 | ** 1: (a, 5) |
1479 | ** 2: (a, 10) |
1480 | ** 3: (a, 10) |
1481 | ** 4: (b) |
1482 | ** 5: (b, 5) |
1483 | ** 6: (c) |
1484 | ** 7: (c, 100) |
1485 | ** 8: (c, 105) |
1486 | ** 9: (c, 105) |
1487 | ** |
1488 | ** For each sample in the aSample[] array, N samples are present in the |
1489 | ** effective sample array. In the above, samples 0 and 1 are based on |
1490 | ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. |
1491 | ** |
1492 | ** Often, sample i of each block of N effective samples has (i+1) fields. |
1493 | ** Except, each sample may be extended to ensure that it is greater than or |
1494 | ** equal to the previous sample in the array. For example, in the above, |
1495 | ** sample 2 is the first sample of a block of N samples, so at first it |
1496 | ** appears that it should be 1 field in size. However, that would make it |
1497 | ** smaller than sample 1, so the binary search would not work. As a result, |
1498 | ** it is extended to two fields. The duplicates that this creates do not |
1499 | ** cause any problems. |
1500 | */ |
1501 | nField = MIN(pRec->nField, pIdx->nSample); |
1502 | iCol = 0; |
1503 | iSample = pIdx->nSample * nField; |
1504 | do{ |
1505 | int iSamp; /* Index in aSample[] of test sample */ |
1506 | int n; /* Number of fields in test sample */ |
1507 | |
1508 | iTest = (iMin+iSample)/2; |
1509 | iSamp = iTest / nField; |
1510 | if( iSamp>0 ){ |
1511 | /* The proposed effective sample is a prefix of sample aSample[iSamp]. |
1512 | ** Specifically, the shortest prefix of at least (1 + iTest%nField) |
1513 | ** fields that is greater than the previous effective sample. */ |
1514 | for(n=(iTest % nField) + 1; n<nField; n++){ |
1515 | if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break; |
1516 | } |
1517 | }else{ |
1518 | n = iTest + 1; |
1519 | } |
1520 | |
1521 | pRec->nField = n; |
1522 | res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); |
1523 | if( res<0 ){ |
1524 | iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; |
1525 | iMin = iTest+1; |
1526 | }else if( res==0 && n<nField ){ |
1527 | iLower = aSample[iSamp].anLt[n-1]; |
1528 | iMin = iTest+1; |
1529 | res = -1; |
1530 | }else{ |
1531 | iSample = iTest; |
1532 | iCol = n-1; |
1533 | } |
1534 | }while( res && iMin<iSample ); |
1535 | i = iSample / nField; |
1536 | |
1537 | #ifdef SQLITE_DEBUG |
1538 | /* The following assert statements check that the binary search code |
1539 | ** above found the right answer. This block serves no purpose other |
1540 | ** than to invoke the asserts. */ |
1541 | if( pParse->db->mallocFailed==0 ){ |
1542 | if( res==0 ){ |
1543 | /* If (res==0) is true, then pRec must be equal to sample i. */ |
1544 | assert( i<pIdx->nSample ); |
1545 | assert( iCol==nField-1 ); |
1546 | pRec->nField = nField; |
1547 | assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) |
1548 | || pParse->db->mallocFailed |
1549 | ); |
1550 | }else{ |
1551 | /* Unless i==pIdx->nSample, indicating that pRec is larger than |
1552 | ** all samples in the aSample[] array, pRec must be smaller than the |
1553 | ** (iCol+1) field prefix of sample i. */ |
1554 | assert( i<=pIdx->nSample && i>=0 ); |
1555 | pRec->nField = iCol+1; |
1556 | assert( i==pIdx->nSample |
1557 | || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 |
1558 | || pParse->db->mallocFailed ); |
1559 | |
1560 | /* if i==0 and iCol==0, then record pRec is smaller than all samples |
1561 | ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must |
1562 | ** be greater than or equal to the (iCol) field prefix of sample i. |
1563 | ** If (i>0), then pRec must also be greater than sample (i-1). */ |
1564 | if( iCol>0 ){ |
1565 | pRec->nField = iCol; |
1566 | assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 |
1567 | || pParse->db->mallocFailed ); |
1568 | } |
1569 | if( i>0 ){ |
1570 | pRec->nField = nField; |
1571 | assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 |
1572 | || pParse->db->mallocFailed ); |
1573 | } |
1574 | } |
1575 | } |
1576 | #endif /* ifdef SQLITE_DEBUG */ |
1577 | |
1578 | if( res==0 ){ |
1579 | /* Record pRec is equal to sample i */ |
1580 | assert( iCol==nField-1 ); |
1581 | aStat[0] = aSample[i].anLt[iCol]; |
1582 | aStat[1] = aSample[i].anEq[iCol]; |
1583 | }else{ |
1584 | /* At this point, the (iCol+1) field prefix of aSample[i] is the first |
1585 | ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec |
1586 | ** is larger than all samples in the array. */ |
1587 | tRowcnt iUpper, iGap; |
1588 | if( i>=pIdx->nSample ){ |
1589 | iUpper = pIdx->nRowEst0; |
1590 | }else{ |
1591 | iUpper = aSample[i].anLt[iCol]; |
1592 | } |
1593 | |
1594 | if( iLower>=iUpper ){ |
1595 | iGap = 0; |
1596 | }else{ |
1597 | iGap = iUpper - iLower; |
1598 | } |
1599 | if( roundUp ){ |
1600 | iGap = (iGap*2)/3; |
1601 | }else{ |
1602 | iGap = iGap/3; |
1603 | } |
1604 | aStat[0] = iLower + iGap; |
1605 | aStat[1] = pIdx->aAvgEq[nField-1]; |
1606 | } |
1607 | |
1608 | /* Restore the pRec->nField value before returning. */ |
1609 | pRec->nField = nField; |
1610 | return i; |
1611 | } |
1612 | #endif /* SQLITE_ENABLE_STAT4 */ |
1613 | |
1614 | /* |
1615 | ** If it is not NULL, pTerm is a term that provides an upper or lower |
1616 | ** bound on a range scan. Without considering pTerm, it is estimated |
1617 | ** that the scan will visit nNew rows. This function returns the number |
1618 | ** estimated to be visited after taking pTerm into account. |
1619 | ** |
1620 | ** If the user explicitly specified a likelihood() value for this term, |
1621 | ** then the return value is the likelihood multiplied by the number of |
1622 | ** input rows. Otherwise, this function assumes that an "IS NOT NULL" term |
1623 | ** has a likelihood of 0.50, and any other term a likelihood of 0.25. |
1624 | */ |
1625 | static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ |
1626 | LogEst nRet = nNew; |
1627 | if( pTerm ){ |
1628 | if( pTerm->truthProb<=0 ){ |
1629 | nRet += pTerm->truthProb; |
1630 | }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ |
1631 | nRet -= 20; assert( 20==sqlite3LogEst(4) ); |
1632 | } |
1633 | } |
1634 | return nRet; |
1635 | } |
1636 | |
1637 | |
1638 | #ifdef SQLITE_ENABLE_STAT4 |
1639 | /* |
1640 | ** Return the affinity for a single column of an index. |
1641 | */ |
1642 | char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ |
1643 | assert( iCol>=0 && iCol<pIdx->nColumn ); |
1644 | if( !pIdx->zColAff ){ |
1645 | if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; |
1646 | } |
1647 | assert( pIdx->zColAff[iCol]!=0 ); |
1648 | return pIdx->zColAff[iCol]; |
1649 | } |
1650 | #endif |
1651 | |
1652 | |
1653 | #ifdef SQLITE_ENABLE_STAT4 |
1654 | /* |
1655 | ** This function is called to estimate the number of rows visited by a |
1656 | ** range-scan on a skip-scan index. For example: |
1657 | ** |
1658 | ** CREATE INDEX i1 ON t1(a, b, c); |
1659 | ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; |
1660 | ** |
1661 | ** Value pLoop->nOut is currently set to the estimated number of rows |
1662 | ** visited for scanning (a=? AND b=?). This function reduces that estimate |
1663 | ** by some factor to account for the (c BETWEEN ? AND ?) expression based |
1664 | ** on the stat4 data for the index. this scan will be peformed multiple |
1665 | ** times (once for each (a,b) combination that matches a=?) is dealt with |
1666 | ** by the caller. |
1667 | ** |
1668 | ** It does this by scanning through all stat4 samples, comparing values |
1669 | ** extracted from pLower and pUpper with the corresponding column in each |
1670 | ** sample. If L and U are the number of samples found to be less than or |
1671 | ** equal to the values extracted from pLower and pUpper respectively, and |
1672 | ** N is the total number of samples, the pLoop->nOut value is adjusted |
1673 | ** as follows: |
1674 | ** |
1675 | ** nOut = nOut * ( min(U - L, 1) / N ) |
1676 | ** |
1677 | ** If pLower is NULL, or a value cannot be extracted from the term, L is |
1678 | ** set to zero. If pUpper is NULL, or a value cannot be extracted from it, |
1679 | ** U is set to N. |
1680 | ** |
1681 | ** Normally, this function sets *pbDone to 1 before returning. However, |
1682 | ** if no value can be extracted from either pLower or pUpper (and so the |
1683 | ** estimate of the number of rows delivered remains unchanged), *pbDone |
1684 | ** is left as is. |
1685 | ** |
1686 | ** If an error occurs, an SQLite error code is returned. Otherwise, |
1687 | ** SQLITE_OK. |
1688 | */ |
1689 | static int whereRangeSkipScanEst( |
1690 | Parse *pParse, /* Parsing & code generating context */ |
1691 | WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ |
1692 | WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ |
1693 | WhereLoop *pLoop, /* Update the .nOut value of this loop */ |
1694 | int *pbDone /* Set to true if at least one expr. value extracted */ |
1695 | ){ |
1696 | Index *p = pLoop->u.btree.pIndex; |
1697 | int nEq = pLoop->u.btree.nEq; |
1698 | sqlite3 *db = pParse->db; |
1699 | int nLower = -1; |
1700 | int nUpper = p->nSample+1; |
1701 | int rc = SQLITE_OK; |
1702 | u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); |
1703 | CollSeq *pColl; |
1704 | |
1705 | sqlite3_value *p1 = 0; /* Value extracted from pLower */ |
1706 | sqlite3_value *p2 = 0; /* Value extracted from pUpper */ |
1707 | sqlite3_value *pVal = 0; /* Value extracted from record */ |
1708 | |
1709 | pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); |
1710 | if( pLower ){ |
1711 | rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); |
1712 | nLower = 0; |
1713 | } |
1714 | if( pUpper && rc==SQLITE_OK ){ |
1715 | rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); |
1716 | nUpper = p2 ? 0 : p->nSample; |
1717 | } |
1718 | |
1719 | if( p1 || p2 ){ |
1720 | int i; |
1721 | int nDiff; |
1722 | for(i=0; rc==SQLITE_OK && i<p->nSample; i++){ |
1723 | rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); |
1724 | if( rc==SQLITE_OK && p1 ){ |
1725 | int res = sqlite3MemCompare(p1, pVal, pColl); |
1726 | if( res>=0 ) nLower++; |
1727 | } |
1728 | if( rc==SQLITE_OK && p2 ){ |
1729 | int res = sqlite3MemCompare(p2, pVal, pColl); |
1730 | if( res>=0 ) nUpper++; |
1731 | } |
1732 | } |
1733 | nDiff = (nUpper - nLower); |
1734 | if( nDiff<=0 ) nDiff = 1; |
1735 | |
1736 | /* If there is both an upper and lower bound specified, and the |
1737 | ** comparisons indicate that they are close together, use the fallback |
1738 | ** method (assume that the scan visits 1/64 of the rows) for estimating |
1739 | ** the number of rows visited. Otherwise, estimate the number of rows |
1740 | ** using the method described in the header comment for this function. */ |
1741 | if( nDiff!=1 || pUpper==0 || pLower==0 ){ |
1742 | int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); |
1743 | pLoop->nOut -= nAdjust; |
1744 | *pbDone = 1; |
1745 | WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n" , |
1746 | nLower, nUpper, nAdjust*-1, pLoop->nOut)); |
1747 | } |
1748 | |
1749 | }else{ |
1750 | assert( *pbDone==0 ); |
1751 | } |
1752 | |
1753 | sqlite3ValueFree(p1); |
1754 | sqlite3ValueFree(p2); |
1755 | sqlite3ValueFree(pVal); |
1756 | |
1757 | return rc; |
1758 | } |
1759 | #endif /* SQLITE_ENABLE_STAT4 */ |
1760 | |
1761 | /* |
1762 | ** This function is used to estimate the number of rows that will be visited |
1763 | ** by scanning an index for a range of values. The range may have an upper |
1764 | ** bound, a lower bound, or both. The WHERE clause terms that set the upper |
1765 | ** and lower bounds are represented by pLower and pUpper respectively. For |
1766 | ** example, assuming that index p is on t1(a): |
1767 | ** |
1768 | ** ... FROM t1 WHERE a > ? AND a < ? ... |
1769 | ** |_____| |_____| |
1770 | ** | | |
1771 | ** pLower pUpper |
1772 | ** |
1773 | ** If either of the upper or lower bound is not present, then NULL is passed in |
1774 | ** place of the corresponding WhereTerm. |
1775 | ** |
1776 | ** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index |
1777 | ** column subject to the range constraint. Or, equivalently, the number of |
1778 | ** equality constraints optimized by the proposed index scan. For example, |
1779 | ** assuming index p is on t1(a, b), and the SQL query is: |
1780 | ** |
1781 | ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... |
1782 | ** |
1783 | ** then nEq is set to 1 (as the range restricted column, b, is the second |
1784 | ** left-most column of the index). Or, if the query is: |
1785 | ** |
1786 | ** ... FROM t1 WHERE a > ? AND a < ? ... |
1787 | ** |
1788 | ** then nEq is set to 0. |
1789 | ** |
1790 | ** When this function is called, *pnOut is set to the sqlite3LogEst() of the |
1791 | ** number of rows that the index scan is expected to visit without |
1792 | ** considering the range constraints. If nEq is 0, then *pnOut is the number of |
1793 | ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) |
1794 | ** to account for the range constraints pLower and pUpper. |
1795 | ** |
1796 | ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be |
1797 | ** used, a single range inequality reduces the search space by a factor of 4. |
1798 | ** and a pair of constraints (x>? AND x<?) reduces the expected number of |
1799 | ** rows visited by a factor of 64. |
1800 | */ |
1801 | static int whereRangeScanEst( |
1802 | Parse *pParse, /* Parsing & code generating context */ |
1803 | WhereLoopBuilder *pBuilder, |
1804 | WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ |
1805 | WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ |
1806 | WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ |
1807 | ){ |
1808 | int rc = SQLITE_OK; |
1809 | int nOut = pLoop->nOut; |
1810 | LogEst nNew; |
1811 | |
1812 | #ifdef SQLITE_ENABLE_STAT4 |
1813 | Index *p = pLoop->u.btree.pIndex; |
1814 | int nEq = pLoop->u.btree.nEq; |
1815 | |
1816 | if( p->nSample>0 && ALWAYS(nEq<p->nSampleCol) |
1817 | && OptimizationEnabled(pParse->db, SQLITE_Stat4) |
1818 | ){ |
1819 | if( nEq==pBuilder->nRecValid ){ |
1820 | UnpackedRecord *pRec = pBuilder->pRec; |
1821 | tRowcnt a[2]; |
1822 | int nBtm = pLoop->u.btree.nBtm; |
1823 | int nTop = pLoop->u.btree.nTop; |
1824 | |
1825 | /* Variable iLower will be set to the estimate of the number of rows in |
1826 | ** the index that are less than the lower bound of the range query. The |
1827 | ** lower bound being the concatenation of $P and $L, where $P is the |
1828 | ** key-prefix formed by the nEq values matched against the nEq left-most |
1829 | ** columns of the index, and $L is the value in pLower. |
1830 | ** |
1831 | ** Or, if pLower is NULL or $L cannot be extracted from it (because it |
1832 | ** is not a simple variable or literal value), the lower bound of the |
1833 | ** range is $P. Due to a quirk in the way whereKeyStats() works, even |
1834 | ** if $L is available, whereKeyStats() is called for both ($P) and |
1835 | ** ($P:$L) and the larger of the two returned values is used. |
1836 | ** |
1837 | ** Similarly, iUpper is to be set to the estimate of the number of rows |
1838 | ** less than the upper bound of the range query. Where the upper bound |
1839 | ** is either ($P) or ($P:$U). Again, even if $U is available, both values |
1840 | ** of iUpper are requested of whereKeyStats() and the smaller used. |
1841 | ** |
1842 | ** The number of rows between the two bounds is then just iUpper-iLower. |
1843 | */ |
1844 | tRowcnt iLower; /* Rows less than the lower bound */ |
1845 | tRowcnt iUpper; /* Rows less than the upper bound */ |
1846 | int iLwrIdx = -2; /* aSample[] for the lower bound */ |
1847 | int iUprIdx = -1; /* aSample[] for the upper bound */ |
1848 | |
1849 | if( pRec ){ |
1850 | testcase( pRec->nField!=pBuilder->nRecValid ); |
1851 | pRec->nField = pBuilder->nRecValid; |
1852 | } |
1853 | /* Determine iLower and iUpper using ($P) only. */ |
1854 | if( nEq==0 ){ |
1855 | iLower = 0; |
1856 | iUpper = p->nRowEst0; |
1857 | }else{ |
1858 | /* Note: this call could be optimized away - since the same values must |
1859 | ** have been requested when testing key $P in whereEqualScanEst(). */ |
1860 | whereKeyStats(pParse, p, pRec, 0, a); |
1861 | iLower = a[0]; |
1862 | iUpper = a[0] + a[1]; |
1863 | } |
1864 | |
1865 | assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); |
1866 | assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); |
1867 | assert( p->aSortOrder!=0 ); |
1868 | if( p->aSortOrder[nEq] ){ |
1869 | /* The roles of pLower and pUpper are swapped for a DESC index */ |
1870 | SWAP(WhereTerm*, pLower, pUpper); |
1871 | SWAP(int, nBtm, nTop); |
1872 | } |
1873 | |
1874 | /* If possible, improve on the iLower estimate using ($P:$L). */ |
1875 | if( pLower ){ |
1876 | int n; /* Values extracted from pExpr */ |
1877 | Expr *pExpr = pLower->pExpr->pRight; |
1878 | rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n); |
1879 | if( rc==SQLITE_OK && n ){ |
1880 | tRowcnt iNew; |
1881 | u16 mask = WO_GT|WO_LE; |
1882 | if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); |
1883 | iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); |
1884 | iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0); |
1885 | if( iNew>iLower ) iLower = iNew; |
1886 | nOut--; |
1887 | pLower = 0; |
1888 | } |
1889 | } |
1890 | |
1891 | /* If possible, improve on the iUpper estimate using ($P:$U). */ |
1892 | if( pUpper ){ |
1893 | int n; /* Values extracted from pExpr */ |
1894 | Expr *pExpr = pUpper->pExpr->pRight; |
1895 | rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n); |
1896 | if( rc==SQLITE_OK && n ){ |
1897 | tRowcnt iNew; |
1898 | u16 mask = WO_GT|WO_LE; |
1899 | if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); |
1900 | iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); |
1901 | iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0); |
1902 | if( iNew<iUpper ) iUpper = iNew; |
1903 | nOut--; |
1904 | pUpper = 0; |
1905 | } |
1906 | } |
1907 | |
1908 | pBuilder->pRec = pRec; |
1909 | if( rc==SQLITE_OK ){ |
1910 | if( iUpper>iLower ){ |
1911 | nNew = sqlite3LogEst(iUpper - iLower); |
1912 | /* TUNING: If both iUpper and iLower are derived from the same |
1913 | ** sample, then assume they are 4x more selective. This brings |
1914 | ** the estimated selectivity more in line with what it would be |
1915 | ** if estimated without the use of STAT4 tables. */ |
1916 | if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); |
1917 | }else{ |
1918 | nNew = 10; assert( 10==sqlite3LogEst(2) ); |
1919 | } |
1920 | if( nNew<nOut ){ |
1921 | nOut = nNew; |
1922 | } |
1923 | WHERETRACE(0x10, ("STAT4 range scan: %u..%u est=%d\n" , |
1924 | (u32)iLower, (u32)iUpper, nOut)); |
1925 | } |
1926 | }else{ |
1927 | int bDone = 0; |
1928 | rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone); |
1929 | if( bDone ) return rc; |
1930 | } |
1931 | } |
1932 | #else |
1933 | UNUSED_PARAMETER(pParse); |
1934 | UNUSED_PARAMETER(pBuilder); |
1935 | assert( pLower || pUpper ); |
1936 | #endif |
1937 | assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); |
1938 | nNew = whereRangeAdjust(pLower, nOut); |
1939 | nNew = whereRangeAdjust(pUpper, nNew); |
1940 | |
1941 | /* TUNING: If there is both an upper and lower limit and neither limit |
1942 | ** has an application-defined likelihood(), assume the range is |
1943 | ** reduced by an additional 75%. This means that, by default, an open-ended |
1944 | ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the |
1945 | ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to |
1946 | ** match 1/64 of the index. */ |
1947 | if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ |
1948 | nNew -= 20; |
1949 | } |
1950 | |
1951 | nOut -= (pLower!=0) + (pUpper!=0); |
1952 | if( nNew<10 ) nNew = 10; |
1953 | if( nNew<nOut ) nOut = nNew; |
1954 | #if defined(WHERETRACE_ENABLED) |
1955 | if( pLoop->nOut>nOut ){ |
1956 | WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n" , |
1957 | pLoop->nOut, nOut)); |
1958 | } |
1959 | #endif |
1960 | pLoop->nOut = (LogEst)nOut; |
1961 | return rc; |
1962 | } |
1963 | |
1964 | #ifdef SQLITE_ENABLE_STAT4 |
1965 | /* |
1966 | ** Estimate the number of rows that will be returned based on |
1967 | ** an equality constraint x=VALUE and where that VALUE occurs in |
1968 | ** the histogram data. This only works when x is the left-most |
1969 | ** column of an index and sqlite_stat4 histogram data is available |
1970 | ** for that index. When pExpr==NULL that means the constraint is |
1971 | ** "x IS NULL" instead of "x=VALUE". |
1972 | ** |
1973 | ** Write the estimated row count into *pnRow and return SQLITE_OK. |
1974 | ** If unable to make an estimate, leave *pnRow unchanged and return |
1975 | ** non-zero. |
1976 | ** |
1977 | ** This routine can fail if it is unable to load a collating sequence |
1978 | ** required for string comparison, or if unable to allocate memory |
1979 | ** for a UTF conversion required for comparison. The error is stored |
1980 | ** in the pParse structure. |
1981 | */ |
1982 | static int whereEqualScanEst( |
1983 | Parse *pParse, /* Parsing & code generating context */ |
1984 | WhereLoopBuilder *pBuilder, |
1985 | Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ |
1986 | tRowcnt *pnRow /* Write the revised row estimate here */ |
1987 | ){ |
1988 | Index *p = pBuilder->pNew->u.btree.pIndex; |
1989 | int nEq = pBuilder->pNew->u.btree.nEq; |
1990 | UnpackedRecord *pRec = pBuilder->pRec; |
1991 | int rc; /* Subfunction return code */ |
1992 | tRowcnt a[2]; /* Statistics */ |
1993 | int bOk; |
1994 | |
1995 | assert( nEq>=1 ); |
1996 | assert( nEq<=p->nColumn ); |
1997 | assert( p->aSample!=0 ); |
1998 | assert( p->nSample>0 ); |
1999 | assert( pBuilder->nRecValid<nEq ); |
2000 | |
2001 | /* If values are not available for all fields of the index to the left |
2002 | ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */ |
2003 | if( pBuilder->nRecValid<(nEq-1) ){ |
2004 | return SQLITE_NOTFOUND; |
2005 | } |
2006 | |
2007 | /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() |
2008 | ** below would return the same value. */ |
2009 | if( nEq>=p->nColumn ){ |
2010 | *pnRow = 1; |
2011 | return SQLITE_OK; |
2012 | } |
2013 | |
2014 | rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk); |
2015 | pBuilder->pRec = pRec; |
2016 | if( rc!=SQLITE_OK ) return rc; |
2017 | if( bOk==0 ) return SQLITE_NOTFOUND; |
2018 | pBuilder->nRecValid = nEq; |
2019 | |
2020 | whereKeyStats(pParse, p, pRec, 0, a); |
2021 | WHERETRACE(0x10,("equality scan regions %s(%d): %d\n" , |
2022 | p->zName, nEq-1, (int)a[1])); |
2023 | *pnRow = a[1]; |
2024 | |
2025 | return rc; |
2026 | } |
2027 | #endif /* SQLITE_ENABLE_STAT4 */ |
2028 | |
2029 | #ifdef SQLITE_ENABLE_STAT4 |
2030 | /* |
2031 | ** Estimate the number of rows that will be returned based on |
2032 | ** an IN constraint where the right-hand side of the IN operator |
2033 | ** is a list of values. Example: |
2034 | ** |
2035 | ** WHERE x IN (1,2,3,4) |
2036 | ** |
2037 | ** Write the estimated row count into *pnRow and return SQLITE_OK. |
2038 | ** If unable to make an estimate, leave *pnRow unchanged and return |
2039 | ** non-zero. |
2040 | ** |
2041 | ** This routine can fail if it is unable to load a collating sequence |
2042 | ** required for string comparison, or if unable to allocate memory |
2043 | ** for a UTF conversion required for comparison. The error is stored |
2044 | ** in the pParse structure. |
2045 | */ |
2046 | static int whereInScanEst( |
2047 | Parse *pParse, /* Parsing & code generating context */ |
2048 | WhereLoopBuilder *pBuilder, |
2049 | ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ |
2050 | tRowcnt *pnRow /* Write the revised row estimate here */ |
2051 | ){ |
2052 | Index *p = pBuilder->pNew->u.btree.pIndex; |
2053 | i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); |
2054 | int nRecValid = pBuilder->nRecValid; |
2055 | int rc = SQLITE_OK; /* Subfunction return code */ |
2056 | tRowcnt nEst; /* Number of rows for a single term */ |
2057 | tRowcnt nRowEst = 0; /* New estimate of the number of rows */ |
2058 | int i; /* Loop counter */ |
2059 | |
2060 | assert( p->aSample!=0 ); |
2061 | for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){ |
2062 | nEst = nRow0; |
2063 | rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); |
2064 | nRowEst += nEst; |
2065 | pBuilder->nRecValid = nRecValid; |
2066 | } |
2067 | |
2068 | if( rc==SQLITE_OK ){ |
2069 | if( nRowEst > nRow0 ) nRowEst = nRow0; |
2070 | *pnRow = nRowEst; |
2071 | WHERETRACE(0x10,("IN row estimate: est=%d\n" , nRowEst)); |
2072 | } |
2073 | assert( pBuilder->nRecValid==nRecValid ); |
2074 | return rc; |
2075 | } |
2076 | #endif /* SQLITE_ENABLE_STAT4 */ |
2077 | |
2078 | |
2079 | #ifdef WHERETRACE_ENABLED |
2080 | /* |
2081 | ** Print the content of a WhereTerm object |
2082 | */ |
2083 | void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ |
2084 | if( pTerm==0 ){ |
2085 | sqlite3DebugPrintf("TERM-%-3d NULL\n" , iTerm); |
2086 | }else{ |
2087 | char zType[8]; |
2088 | char zLeft[50]; |
2089 | memcpy(zType, "...." , 5); |
2090 | if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; |
2091 | if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; |
2092 | if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L'; |
2093 | if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C'; |
2094 | if( pTerm->eOperator & WO_SINGLE ){ |
2095 | assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); |
2096 | sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}" , |
2097 | pTerm->leftCursor, pTerm->u.x.leftColumn); |
2098 | }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){ |
2099 | sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx" , |
2100 | pTerm->u.pOrInfo->indexable); |
2101 | }else{ |
2102 | sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d" , pTerm->leftCursor); |
2103 | } |
2104 | sqlite3DebugPrintf( |
2105 | "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x" , |
2106 | iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags); |
2107 | /* The 0x10000 .wheretrace flag causes extra information to be |
2108 | ** shown about each Term */ |
2109 | if( sqlite3WhereTrace & 0x10000 ){ |
2110 | sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx" , |
2111 | pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight); |
2112 | } |
2113 | if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){ |
2114 | sqlite3DebugPrintf(" iField=%d" , pTerm->u.x.iField); |
2115 | } |
2116 | if( pTerm->iParent>=0 ){ |
2117 | sqlite3DebugPrintf(" iParent=%d" , pTerm->iParent); |
2118 | } |
2119 | sqlite3DebugPrintf("\n" ); |
2120 | sqlite3TreeViewExpr(0, pTerm->pExpr, 0); |
2121 | } |
2122 | } |
2123 | #endif |
2124 | |
2125 | #ifdef WHERETRACE_ENABLED |
2126 | /* |
2127 | ** Show the complete content of a WhereClause |
2128 | */ |
2129 | void sqlite3WhereClausePrint(WhereClause *pWC){ |
2130 | int i; |
2131 | for(i=0; i<pWC->nTerm; i++){ |
2132 | sqlite3WhereTermPrint(&pWC->a[i], i); |
2133 | } |
2134 | } |
2135 | #endif |
2136 | |
2137 | #ifdef WHERETRACE_ENABLED |
2138 | /* |
2139 | ** Print a WhereLoop object for debugging purposes |
2140 | */ |
2141 | void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ |
2142 | WhereInfo *pWInfo = pWC->pWInfo; |
2143 | int nb = 1+(pWInfo->pTabList->nSrc+3)/4; |
2144 | SrcItem *pItem = pWInfo->pTabList->a + p->iTab; |
2145 | Table *pTab = pItem->pTab; |
2146 | Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; |
2147 | sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx" , p->cId, |
2148 | p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); |
2149 | sqlite3DebugPrintf(" %12s" , |
2150 | pItem->zAlias ? pItem->zAlias : pTab->zName); |
2151 | if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ |
2152 | const char *zName; |
2153 | if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ |
2154 | if( strncmp(zName, "sqlite_autoindex_" , 17)==0 ){ |
2155 | int i = sqlite3Strlen30(zName) - 1; |
2156 | while( zName[i]!='_' ) i--; |
2157 | zName += i; |
2158 | } |
2159 | sqlite3DebugPrintf(".%-16s %2d" , zName, p->u.btree.nEq); |
2160 | }else{ |
2161 | sqlite3DebugPrintf("%20s" ,"" ); |
2162 | } |
2163 | }else{ |
2164 | char *z; |
2165 | if( p->u.vtab.idxStr ){ |
2166 | z = sqlite3_mprintf("(%d,\"%s\",%#x)" , |
2167 | p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); |
2168 | }else{ |
2169 | z = sqlite3_mprintf("(%d,%x)" , p->u.vtab.idxNum, p->u.vtab.omitMask); |
2170 | } |
2171 | sqlite3DebugPrintf(" %-19s" , z); |
2172 | sqlite3_free(z); |
2173 | } |
2174 | if( p->wsFlags & WHERE_SKIPSCAN ){ |
2175 | sqlite3DebugPrintf(" f %06x %d-%d" , p->wsFlags, p->nLTerm,p->nSkip); |
2176 | }else{ |
2177 | sqlite3DebugPrintf(" f %06x N %d" , p->wsFlags, p->nLTerm); |
2178 | } |
2179 | sqlite3DebugPrintf(" cost %d,%d,%d\n" , p->rSetup, p->rRun, p->nOut); |
2180 | if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ |
2181 | int i; |
2182 | for(i=0; i<p->nLTerm; i++){ |
2183 | sqlite3WhereTermPrint(p->aLTerm[i], i); |
2184 | } |
2185 | } |
2186 | } |
2187 | #endif |
2188 | |
2189 | /* |
2190 | ** Convert bulk memory into a valid WhereLoop that can be passed |
2191 | ** to whereLoopClear harmlessly. |
2192 | */ |
2193 | static void whereLoopInit(WhereLoop *p){ |
2194 | p->aLTerm = p->aLTermSpace; |
2195 | p->nLTerm = 0; |
2196 | p->nLSlot = ArraySize(p->aLTermSpace); |
2197 | p->wsFlags = 0; |
2198 | } |
2199 | |
2200 | /* |
2201 | ** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. |
2202 | */ |
2203 | static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ |
2204 | if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ |
2205 | if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ |
2206 | sqlite3_free(p->u.vtab.idxStr); |
2207 | p->u.vtab.needFree = 0; |
2208 | p->u.vtab.idxStr = 0; |
2209 | }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ |
2210 | sqlite3DbFree(db, p->u.btree.pIndex->zColAff); |
2211 | sqlite3DbFreeNN(db, p->u.btree.pIndex); |
2212 | p->u.btree.pIndex = 0; |
2213 | } |
2214 | } |
2215 | } |
2216 | |
2217 | /* |
2218 | ** Deallocate internal memory used by a WhereLoop object. Leave the |
2219 | ** object in an initialized state, as if it had been newly allocated. |
2220 | */ |
2221 | static void whereLoopClear(sqlite3 *db, WhereLoop *p){ |
2222 | if( p->aLTerm!=p->aLTermSpace ){ |
2223 | sqlite3DbFreeNN(db, p->aLTerm); |
2224 | p->aLTerm = p->aLTermSpace; |
2225 | p->nLSlot = ArraySize(p->aLTermSpace); |
2226 | } |
2227 | whereLoopClearUnion(db, p); |
2228 | p->nLTerm = 0; |
2229 | p->wsFlags = 0; |
2230 | } |
2231 | |
2232 | /* |
2233 | ** Increase the memory allocation for pLoop->aLTerm[] to be at least n. |
2234 | */ |
2235 | static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ |
2236 | WhereTerm **paNew; |
2237 | if( p->nLSlot>=n ) return SQLITE_OK; |
2238 | n = (n+7)&~7; |
2239 | paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n); |
2240 | if( paNew==0 ) return SQLITE_NOMEM_BKPT; |
2241 | memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); |
2242 | if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); |
2243 | p->aLTerm = paNew; |
2244 | p->nLSlot = n; |
2245 | return SQLITE_OK; |
2246 | } |
2247 | |
2248 | /* |
2249 | ** Transfer content from the second pLoop into the first. |
2250 | */ |
2251 | static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ |
2252 | whereLoopClearUnion(db, pTo); |
2253 | if( pFrom->nLTerm > pTo->nLSlot |
2254 | && whereLoopResize(db, pTo, pFrom->nLTerm) |
2255 | ){ |
2256 | memset(pTo, 0, WHERE_LOOP_XFER_SZ); |
2257 | return SQLITE_NOMEM_BKPT; |
2258 | } |
2259 | memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); |
2260 | memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); |
2261 | if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ |
2262 | pFrom->u.vtab.needFree = 0; |
2263 | }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ |
2264 | pFrom->u.btree.pIndex = 0; |
2265 | } |
2266 | return SQLITE_OK; |
2267 | } |
2268 | |
2269 | /* |
2270 | ** Delete a WhereLoop object |
2271 | */ |
2272 | static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ |
2273 | assert( db!=0 ); |
2274 | whereLoopClear(db, p); |
2275 | sqlite3DbNNFreeNN(db, p); |
2276 | } |
2277 | |
2278 | /* |
2279 | ** Free a WhereInfo structure |
2280 | */ |
2281 | static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ |
2282 | assert( pWInfo!=0 ); |
2283 | assert( db!=0 ); |
2284 | sqlite3WhereClauseClear(&pWInfo->sWC); |
2285 | while( pWInfo->pLoops ){ |
2286 | WhereLoop *p = pWInfo->pLoops; |
2287 | pWInfo->pLoops = p->pNextLoop; |
2288 | whereLoopDelete(db, p); |
2289 | } |
2290 | while( pWInfo->pMemToFree ){ |
2291 | WhereMemBlock *pNext = pWInfo->pMemToFree->pNext; |
2292 | sqlite3DbNNFreeNN(db, pWInfo->pMemToFree); |
2293 | pWInfo->pMemToFree = pNext; |
2294 | } |
2295 | sqlite3DbNNFreeNN(db, pWInfo); |
2296 | } |
2297 | |
2298 | /* |
2299 | ** Return TRUE if all of the following are true: |
2300 | ** |
2301 | ** (1) X has the same or lower cost, or returns the same or fewer rows, |
2302 | ** than Y. |
2303 | ** (2) X uses fewer WHERE clause terms than Y |
2304 | ** (3) Every WHERE clause term used by X is also used by Y |
2305 | ** (4) X skips at least as many columns as Y |
2306 | ** (5) If X is a covering index, than Y is too |
2307 | ** |
2308 | ** Conditions (2) and (3) mean that X is a "proper subset" of Y. |
2309 | ** If X is a proper subset of Y then Y is a better choice and ought |
2310 | ** to have a lower cost. This routine returns TRUE when that cost |
2311 | ** relationship is inverted and needs to be adjusted. Constraint (4) |
2312 | ** was added because if X uses skip-scan less than Y it still might |
2313 | ** deserve a lower cost even if it is a proper subset of Y. Constraint (5) |
2314 | ** was added because a covering index probably deserves to have a lower cost |
2315 | ** than a non-covering index even if it is a proper subset. |
2316 | */ |
2317 | static int whereLoopCheaperProperSubset( |
2318 | const WhereLoop *pX, /* First WhereLoop to compare */ |
2319 | const WhereLoop *pY /* Compare against this WhereLoop */ |
2320 | ){ |
2321 | int i, j; |
2322 | if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ |
2323 | return 0; /* X is not a subset of Y */ |
2324 | } |
2325 | if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; |
2326 | if( pY->nSkip > pX->nSkip ) return 0; |
2327 | for(i=pX->nLTerm-1; i>=0; i--){ |
2328 | if( pX->aLTerm[i]==0 ) continue; |
2329 | for(j=pY->nLTerm-1; j>=0; j--){ |
2330 | if( pY->aLTerm[j]==pX->aLTerm[i] ) break; |
2331 | } |
2332 | if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ |
2333 | } |
2334 | if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 |
2335 | && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ |
2336 | return 0; /* Constraint (5) */ |
2337 | } |
2338 | return 1; /* All conditions meet */ |
2339 | } |
2340 | |
2341 | /* |
2342 | ** Try to adjust the cost and number of output rows of WhereLoop pTemplate |
2343 | ** upwards or downwards so that: |
2344 | ** |
2345 | ** (1) pTemplate costs less than any other WhereLoops that are a proper |
2346 | ** subset of pTemplate |
2347 | ** |
2348 | ** (2) pTemplate costs more than any other WhereLoops for which pTemplate |
2349 | ** is a proper subset. |
2350 | ** |
2351 | ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer |
2352 | ** WHERE clause terms than Y and that every WHERE clause term used by X is |
2353 | ** also used by Y. |
2354 | */ |
2355 | static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ |
2356 | if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; |
2357 | for(; p; p=p->pNextLoop){ |
2358 | if( p->iTab!=pTemplate->iTab ) continue; |
2359 | if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; |
2360 | if( whereLoopCheaperProperSubset(p, pTemplate) ){ |
2361 | /* Adjust pTemplate cost downward so that it is cheaper than its |
2362 | ** subset p. */ |
2363 | WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n" , |
2364 | pTemplate->rRun, pTemplate->nOut, |
2365 | MIN(p->rRun, pTemplate->rRun), |
2366 | MIN(p->nOut - 1, pTemplate->nOut))); |
2367 | pTemplate->rRun = MIN(p->rRun, pTemplate->rRun); |
2368 | pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut); |
2369 | }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ |
2370 | /* Adjust pTemplate cost upward so that it is costlier than p since |
2371 | ** pTemplate is a proper subset of p */ |
2372 | WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n" , |
2373 | pTemplate->rRun, pTemplate->nOut, |
2374 | MAX(p->rRun, pTemplate->rRun), |
2375 | MAX(p->nOut + 1, pTemplate->nOut))); |
2376 | pTemplate->rRun = MAX(p->rRun, pTemplate->rRun); |
2377 | pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut); |
2378 | } |
2379 | } |
2380 | } |
2381 | |
2382 | /* |
2383 | ** Search the list of WhereLoops in *ppPrev looking for one that can be |
2384 | ** replaced by pTemplate. |
2385 | ** |
2386 | ** Return NULL if pTemplate does not belong on the WhereLoop list. |
2387 | ** In other words if pTemplate ought to be dropped from further consideration. |
2388 | ** |
2389 | ** If pX is a WhereLoop that pTemplate can replace, then return the |
2390 | ** link that points to pX. |
2391 | ** |
2392 | ** If pTemplate cannot replace any existing element of the list but needs |
2393 | ** to be added to the list as a new entry, then return a pointer to the |
2394 | ** tail of the list. |
2395 | */ |
2396 | static WhereLoop **whereLoopFindLesser( |
2397 | WhereLoop **ppPrev, |
2398 | const WhereLoop *pTemplate |
2399 | ){ |
2400 | WhereLoop *p; |
2401 | for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ |
2402 | if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ |
2403 | /* If either the iTab or iSortIdx values for two WhereLoop are different |
2404 | ** then those WhereLoops need to be considered separately. Neither is |
2405 | ** a candidate to replace the other. */ |
2406 | continue; |
2407 | } |
2408 | /* In the current implementation, the rSetup value is either zero |
2409 | ** or the cost of building an automatic index (NlogN) and the NlogN |
2410 | ** is the same for compatible WhereLoops. */ |
2411 | assert( p->rSetup==0 || pTemplate->rSetup==0 |
2412 | || p->rSetup==pTemplate->rSetup ); |
2413 | |
2414 | /* whereLoopAddBtree() always generates and inserts the automatic index |
2415 | ** case first. Hence compatible candidate WhereLoops never have a larger |
2416 | ** rSetup. Call this SETUP-INVARIANT */ |
2417 | assert( p->rSetup>=pTemplate->rSetup ); |
2418 | |
2419 | /* Any loop using an appliation-defined index (or PRIMARY KEY or |
2420 | ** UNIQUE constraint) with one or more == constraints is better |
2421 | ** than an automatic index. Unless it is a skip-scan. */ |
2422 | if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 |
2423 | && (pTemplate->nSkip)==0 |
2424 | && (pTemplate->wsFlags & WHERE_INDEXED)!=0 |
2425 | && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 |
2426 | && (p->prereq & pTemplate->prereq)==pTemplate->prereq |
2427 | ){ |
2428 | break; |
2429 | } |
2430 | |
2431 | /* If existing WhereLoop p is better than pTemplate, pTemplate can be |
2432 | ** discarded. WhereLoop p is better if: |
2433 | ** (1) p has no more dependencies than pTemplate, and |
2434 | ** (2) p has an equal or lower cost than pTemplate |
2435 | */ |
2436 | if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ |
2437 | && p->rSetup<=pTemplate->rSetup /* (2a) */ |
2438 | && p->rRun<=pTemplate->rRun /* (2b) */ |
2439 | && p->nOut<=pTemplate->nOut /* (2c) */ |
2440 | ){ |
2441 | return 0; /* Discard pTemplate */ |
2442 | } |
2443 | |
2444 | /* If pTemplate is always better than p, then cause p to be overwritten |
2445 | ** with pTemplate. pTemplate is better than p if: |
2446 | ** (1) pTemplate has no more dependences than p, and |
2447 | ** (2) pTemplate has an equal or lower cost than p. |
2448 | */ |
2449 | if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ |
2450 | && p->rRun>=pTemplate->rRun /* (2a) */ |
2451 | && p->nOut>=pTemplate->nOut /* (2b) */ |
2452 | ){ |
2453 | assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ |
2454 | break; /* Cause p to be overwritten by pTemplate */ |
2455 | } |
2456 | } |
2457 | return ppPrev; |
2458 | } |
2459 | |
2460 | /* |
2461 | ** Insert or replace a WhereLoop entry using the template supplied. |
2462 | ** |
2463 | ** An existing WhereLoop entry might be overwritten if the new template |
2464 | ** is better and has fewer dependencies. Or the template will be ignored |
2465 | ** and no insert will occur if an existing WhereLoop is faster and has |
2466 | ** fewer dependencies than the template. Otherwise a new WhereLoop is |
2467 | ** added based on the template. |
2468 | ** |
2469 | ** If pBuilder->pOrSet is not NULL then we care about only the |
2470 | ** prerequisites and rRun and nOut costs of the N best loops. That |
2471 | ** information is gathered in the pBuilder->pOrSet object. This special |
2472 | ** processing mode is used only for OR clause processing. |
2473 | ** |
2474 | ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we |
2475 | ** still might overwrite similar loops with the new template if the |
2476 | ** new template is better. Loops may be overwritten if the following |
2477 | ** conditions are met: |
2478 | ** |
2479 | ** (1) They have the same iTab. |
2480 | ** (2) They have the same iSortIdx. |
2481 | ** (3) The template has same or fewer dependencies than the current loop |
2482 | ** (4) The template has the same or lower cost than the current loop |
2483 | */ |
2484 | static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ |
2485 | WhereLoop **ppPrev, *p; |
2486 | WhereInfo *pWInfo = pBuilder->pWInfo; |
2487 | sqlite3 *db = pWInfo->pParse->db; |
2488 | int rc; |
2489 | |
2490 | /* Stop the search once we hit the query planner search limit */ |
2491 | if( pBuilder->iPlanLimit==0 ){ |
2492 | WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n" )); |
2493 | if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0; |
2494 | return SQLITE_DONE; |
2495 | } |
2496 | pBuilder->iPlanLimit--; |
2497 | |
2498 | whereLoopAdjustCost(pWInfo->pLoops, pTemplate); |
2499 | |
2500 | /* If pBuilder->pOrSet is defined, then only keep track of the costs |
2501 | ** and prereqs. |
2502 | */ |
2503 | if( pBuilder->pOrSet!=0 ){ |
2504 | if( pTemplate->nLTerm ){ |
2505 | #if WHERETRACE_ENABLED |
2506 | u16 n = pBuilder->pOrSet->n; |
2507 | int x = |
2508 | #endif |
2509 | whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, |
2510 | pTemplate->nOut); |
2511 | #if WHERETRACE_ENABLED /* 0x8 */ |
2512 | if( sqlite3WhereTrace & 0x8 ){ |
2513 | sqlite3DebugPrintf(x?" or-%d: " :" or-X: " , n); |
2514 | sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); |
2515 | } |
2516 | #endif |
2517 | } |
2518 | return SQLITE_OK; |
2519 | } |
2520 | |
2521 | /* Look for an existing WhereLoop to replace with pTemplate |
2522 | */ |
2523 | ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); |
2524 | |
2525 | if( ppPrev==0 ){ |
2526 | /* There already exists a WhereLoop on the list that is better |
2527 | ** than pTemplate, so just ignore pTemplate */ |
2528 | #if WHERETRACE_ENABLED /* 0x8 */ |
2529 | if( sqlite3WhereTrace & 0x8 ){ |
2530 | sqlite3DebugPrintf(" skip: " ); |
2531 | sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); |
2532 | } |
2533 | #endif |
2534 | return SQLITE_OK; |
2535 | }else{ |
2536 | p = *ppPrev; |
2537 | } |
2538 | |
2539 | /* If we reach this point it means that either p[] should be overwritten |
2540 | ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new |
2541 | ** WhereLoop and insert it. |
2542 | */ |
2543 | #if WHERETRACE_ENABLED /* 0x8 */ |
2544 | if( sqlite3WhereTrace & 0x8 ){ |
2545 | if( p!=0 ){ |
2546 | sqlite3DebugPrintf("replace: " ); |
2547 | sqlite3WhereLoopPrint(p, pBuilder->pWC); |
2548 | sqlite3DebugPrintf(" with: " ); |
2549 | }else{ |
2550 | sqlite3DebugPrintf(" add: " ); |
2551 | } |
2552 | sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); |
2553 | } |
2554 | #endif |
2555 | if( p==0 ){ |
2556 | /* Allocate a new WhereLoop to add to the end of the list */ |
2557 | *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop)); |
2558 | if( p==0 ) return SQLITE_NOMEM_BKPT; |
2559 | whereLoopInit(p); |
2560 | p->pNextLoop = 0; |
2561 | }else{ |
2562 | /* We will be overwriting WhereLoop p[]. But before we do, first |
2563 | ** go through the rest of the list and delete any other entries besides |
2564 | ** p[] that are also supplated by pTemplate */ |
2565 | WhereLoop **ppTail = &p->pNextLoop; |
2566 | WhereLoop *pToDel; |
2567 | while( *ppTail ){ |
2568 | ppTail = whereLoopFindLesser(ppTail, pTemplate); |
2569 | if( ppTail==0 ) break; |
2570 | pToDel = *ppTail; |
2571 | if( pToDel==0 ) break; |
2572 | *ppTail = pToDel->pNextLoop; |
2573 | #if WHERETRACE_ENABLED /* 0x8 */ |
2574 | if( sqlite3WhereTrace & 0x8 ){ |
2575 | sqlite3DebugPrintf(" delete: " ); |
2576 | sqlite3WhereLoopPrint(pToDel, pBuilder->pWC); |
2577 | } |
2578 | #endif |
2579 | whereLoopDelete(db, pToDel); |
2580 | } |
2581 | } |
2582 | rc = whereLoopXfer(db, p, pTemplate); |
2583 | if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ |
2584 | Index *pIndex = p->u.btree.pIndex; |
2585 | if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){ |
2586 | p->u.btree.pIndex = 0; |
2587 | } |
2588 | } |
2589 | return rc; |
2590 | } |
2591 | |
2592 | /* |
2593 | ** Adjust the WhereLoop.nOut value downward to account for terms of the |
2594 | ** WHERE clause that reference the loop but which are not used by an |
2595 | ** index. |
2596 | * |
2597 | ** For every WHERE clause term that is not used by the index |
2598 | ** and which has a truth probability assigned by one of the likelihood(), |
2599 | ** likely(), or unlikely() SQL functions, reduce the estimated number |
2600 | ** of output rows by the probability specified. |
2601 | ** |
2602 | ** TUNING: For every WHERE clause term that is not used by the index |
2603 | ** and which does not have an assigned truth probability, heuristics |
2604 | ** described below are used to try to estimate the truth probability. |
2605 | ** TODO --> Perhaps this is something that could be improved by better |
2606 | ** table statistics. |
2607 | ** |
2608 | ** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% |
2609 | ** value corresponds to -1 in LogEst notation, so this means decrement |
2610 | ** the WhereLoop.nOut field for every such WHERE clause term. |
2611 | ** |
2612 | ** Heuristic 2: If there exists one or more WHERE clause terms of the |
2613 | ** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the |
2614 | ** final output row estimate is no greater than 1/4 of the total number |
2615 | ** of rows in the table. In other words, assume that x==EXPR will filter |
2616 | ** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the |
2617 | ** "x" column is boolean or else -1 or 0 or 1 is a common default value |
2618 | ** on the "x" column and so in that case only cap the output row estimate |
2619 | ** at 1/2 instead of 1/4. |
2620 | */ |
2621 | static void whereLoopOutputAdjust( |
2622 | WhereClause *pWC, /* The WHERE clause */ |
2623 | WhereLoop *pLoop, /* The loop to adjust downward */ |
2624 | LogEst nRow /* Number of rows in the entire table */ |
2625 | ){ |
2626 | WhereTerm *pTerm, *pX; |
2627 | Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); |
2628 | int i, j; |
2629 | LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ |
2630 | |
2631 | assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); |
2632 | for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ |
2633 | assert( pTerm!=0 ); |
2634 | if( (pTerm->prereqAll & notAllowed)!=0 ) continue; |
2635 | if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; |
2636 | if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue; |
2637 | for(j=pLoop->nLTerm-1; j>=0; j--){ |
2638 | pX = pLoop->aLTerm[j]; |
2639 | if( pX==0 ) continue; |
2640 | if( pX==pTerm ) break; |
2641 | if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; |
2642 | } |
2643 | if( j<0 ){ |
2644 | if( pLoop->maskSelf==pTerm->prereqAll ){ |
2645 | /* If there are extra terms in the WHERE clause not used by an index |
2646 | ** that depend only on the table being scanned, and that will tend to |
2647 | ** cause many rows to be omitted, then mark that table as |
2648 | ** "self-culling". |
2649 | ** |
2650 | ** 2022-03-24: Self-culling only applies if either the extra terms |
2651 | ** are straight comparison operators that are non-true with NULL |
2652 | ** operand, or if the loop is not an OUTER JOIN. |
2653 | */ |
2654 | if( (pTerm->eOperator & 0x3f)!=0 |
2655 | || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype |
2656 | & (JT_LEFT|JT_LTORJ))==0 |
2657 | ){ |
2658 | pLoop->wsFlags |= WHERE_SELFCULL; |
2659 | } |
2660 | } |
2661 | if( pTerm->truthProb<=0 ){ |
2662 | /* If a truth probability is specified using the likelihood() hints, |
2663 | ** then use the probability provided by the application. */ |
2664 | pLoop->nOut += pTerm->truthProb; |
2665 | }else{ |
2666 | /* In the absence of explicit truth probabilities, use heuristics to |
2667 | ** guess a reasonable truth probability. */ |
2668 | pLoop->nOut--; |
2669 | if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 |
2670 | && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ |
2671 | ){ |
2672 | Expr *pRight = pTerm->pExpr->pRight; |
2673 | int k = 0; |
2674 | testcase( pTerm->pExpr->op==TK_IS ); |
2675 | if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ |
2676 | k = 10; |
2677 | }else{ |
2678 | k = 20; |
2679 | } |
2680 | if( iReduce<k ){ |
2681 | pTerm->wtFlags |= TERM_HEURTRUTH; |
2682 | iReduce = k; |
2683 | } |
2684 | } |
2685 | } |
2686 | } |
2687 | } |
2688 | if( pLoop->nOut > nRow-iReduce ){ |
2689 | pLoop->nOut = nRow - iReduce; |
2690 | } |
2691 | } |
2692 | |
2693 | /* |
2694 | ** Term pTerm is a vector range comparison operation. The first comparison |
2695 | ** in the vector can be optimized using column nEq of the index. This |
2696 | ** function returns the total number of vector elements that can be used |
2697 | ** as part of the range comparison. |
2698 | ** |
2699 | ** For example, if the query is: |
2700 | ** |
2701 | ** WHERE a = ? AND (b, c, d) > (?, ?, ?) |
2702 | ** |
2703 | ** and the index: |
2704 | ** |
2705 | ** CREATE INDEX ... ON (a, b, c, d, e) |
2706 | ** |
2707 | ** then this function would be invoked with nEq=1. The value returned in |
2708 | ** this case is 3. |
2709 | */ |
2710 | static int whereRangeVectorLen( |
2711 | Parse *pParse, /* Parsing context */ |
2712 | int iCur, /* Cursor open on pIdx */ |
2713 | Index *pIdx, /* The index to be used for a inequality constraint */ |
2714 | int nEq, /* Number of prior equality constraints on same index */ |
2715 | WhereTerm *pTerm /* The vector inequality constraint */ |
2716 | ){ |
2717 | int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft); |
2718 | int i; |
2719 | |
2720 | nCmp = MIN(nCmp, (pIdx->nColumn - nEq)); |
2721 | for(i=1; i<nCmp; i++){ |
2722 | /* Test if comparison i of pTerm is compatible with column (i+nEq) |
2723 | ** of the index. If not, exit the loop. */ |
2724 | char aff; /* Comparison affinity */ |
2725 | char idxaff = 0; /* Indexed columns affinity */ |
2726 | CollSeq *pColl; /* Comparison collation sequence */ |
2727 | Expr *pLhs, *pRhs; |
2728 | |
2729 | assert( ExprUseXList(pTerm->pExpr->pLeft) ); |
2730 | pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr; |
2731 | pRhs = pTerm->pExpr->pRight; |
2732 | if( ExprUseXSelect(pRhs) ){ |
2733 | pRhs = pRhs->x.pSelect->pEList->a[i].pExpr; |
2734 | }else{ |
2735 | pRhs = pRhs->x.pList->a[i].pExpr; |
2736 | } |
2737 | |
2738 | /* Check that the LHS of the comparison is a column reference to |
2739 | ** the right column of the right source table. And that the sort |
2740 | ** order of the index column is the same as the sort order of the |
2741 | ** leftmost index column. */ |
2742 | if( pLhs->op!=TK_COLUMN |
2743 | || pLhs->iTable!=iCur |
2744 | || pLhs->iColumn!=pIdx->aiColumn[i+nEq] |
2745 | || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq] |
2746 | ){ |
2747 | break; |
2748 | } |
2749 | |
2750 | testcase( pLhs->iColumn==XN_ROWID ); |
2751 | aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs)); |
2752 | idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); |
2753 | if( aff!=idxaff ) break; |
2754 | |
2755 | pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); |
2756 | if( pColl==0 ) break; |
2757 | if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; |
2758 | } |
2759 | return i; |
2760 | } |
2761 | |
2762 | /* |
2763 | ** Adjust the cost C by the costMult facter T. This only occurs if |
2764 | ** compiled with -DSQLITE_ENABLE_COSTMULT |
2765 | */ |
2766 | #ifdef SQLITE_ENABLE_COSTMULT |
2767 | # define ApplyCostMultiplier(C,T) C += T |
2768 | #else |
2769 | # define ApplyCostMultiplier(C,T) |
2770 | #endif |
2771 | |
2772 | /* |
2773 | ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the |
2774 | ** index pIndex. Try to match one more. |
2775 | ** |
2776 | ** When this function is called, pBuilder->pNew->nOut contains the |
2777 | ** number of rows expected to be visited by filtering using the nEq |
2778 | ** terms only. If it is modified, this value is restored before this |
2779 | ** function returns. |
2780 | ** |
2781 | ** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is |
2782 | ** a fake index used for the INTEGER PRIMARY KEY. |
2783 | */ |
2784 | static int whereLoopAddBtreeIndex( |
2785 | WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ |
2786 | SrcItem *pSrc, /* FROM clause term being analyzed */ |
2787 | Index *pProbe, /* An index on pSrc */ |
2788 | LogEst nInMul /* log(Number of iterations due to IN) */ |
2789 | ){ |
2790 | WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ |
2791 | Parse *pParse = pWInfo->pParse; /* Parsing context */ |
2792 | sqlite3 *db = pParse->db; /* Database connection malloc context */ |
2793 | WhereLoop *pNew; /* Template WhereLoop under construction */ |
2794 | WhereTerm *pTerm; /* A WhereTerm under consideration */ |
2795 | int opMask; /* Valid operators for constraints */ |
2796 | WhereScan scan; /* Iterator for WHERE terms */ |
2797 | Bitmask saved_prereq; /* Original value of pNew->prereq */ |
2798 | u16 saved_nLTerm; /* Original value of pNew->nLTerm */ |
2799 | u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ |
2800 | u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */ |
2801 | u16 saved_nTop; /* Original value of pNew->u.btree.nTop */ |
2802 | u16 saved_nSkip; /* Original value of pNew->nSkip */ |
2803 | u32 saved_wsFlags; /* Original value of pNew->wsFlags */ |
2804 | LogEst saved_nOut; /* Original value of pNew->nOut */ |
2805 | int rc = SQLITE_OK; /* Return code */ |
2806 | LogEst rSize; /* Number of rows in the table */ |
2807 | LogEst rLogSize; /* Logarithm of table size */ |
2808 | WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ |
2809 | |
2810 | pNew = pBuilder->pNew; |
2811 | if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; |
2812 | WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n" , |
2813 | pProbe->pTable->zName,pProbe->zName, |
2814 | pNew->u.btree.nEq, pNew->nSkip, pNew->rRun)); |
2815 | |
2816 | assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); |
2817 | assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); |
2818 | if( pNew->wsFlags & WHERE_BTM_LIMIT ){ |
2819 | opMask = WO_LT|WO_LE; |
2820 | }else{ |
2821 | assert( pNew->u.btree.nBtm==0 ); |
2822 | opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; |
2823 | } |
2824 | if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); |
2825 | |
2826 | assert( pNew->u.btree.nEq<pProbe->nColumn ); |
2827 | assert( pNew->u.btree.nEq<pProbe->nKeyCol |
2828 | || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY ); |
2829 | |
2830 | saved_nEq = pNew->u.btree.nEq; |
2831 | saved_nBtm = pNew->u.btree.nBtm; |
2832 | saved_nTop = pNew->u.btree.nTop; |
2833 | saved_nSkip = pNew->nSkip; |
2834 | saved_nLTerm = pNew->nLTerm; |
2835 | saved_wsFlags = pNew->wsFlags; |
2836 | saved_prereq = pNew->prereq; |
2837 | saved_nOut = pNew->nOut; |
2838 | pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, |
2839 | opMask, pProbe); |
2840 | pNew->rSetup = 0; |
2841 | rSize = pProbe->aiRowLogEst[0]; |
2842 | rLogSize = estLog(rSize); |
2843 | for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ |
2844 | u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ |
2845 | LogEst rCostIdx; |
2846 | LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ |
2847 | int nIn = 0; |
2848 | #ifdef SQLITE_ENABLE_STAT4 |
2849 | int nRecValid = pBuilder->nRecValid; |
2850 | #endif |
2851 | if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) |
2852 | && indexColumnNotNull(pProbe, saved_nEq) |
2853 | ){ |
2854 | continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ |
2855 | } |
2856 | if( pTerm->prereqRight & pNew->maskSelf ) continue; |
2857 | |
2858 | /* Do not allow the upper bound of a LIKE optimization range constraint |
2859 | ** to mix with a lower range bound from some other source */ |
2860 | if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; |
2861 | |
2862 | if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 |
2863 | && !constraintCompatibleWithOuterJoin(pTerm,pSrc) |
2864 | ){ |
2865 | continue; |
2866 | } |
2867 | if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ |
2868 | pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; |
2869 | }else{ |
2870 | pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED; |
2871 | } |
2872 | pNew->wsFlags = saved_wsFlags; |
2873 | pNew->u.btree.nEq = saved_nEq; |
2874 | pNew->u.btree.nBtm = saved_nBtm; |
2875 | pNew->u.btree.nTop = saved_nTop; |
2876 | pNew->nLTerm = saved_nLTerm; |
2877 | if( pNew->nLTerm>=pNew->nLSlot |
2878 | && whereLoopResize(db, pNew, pNew->nLTerm+1) |
2879 | ){ |
2880 | break; /* OOM while trying to enlarge the pNew->aLTerm array */ |
2881 | } |
2882 | pNew->aLTerm[pNew->nLTerm++] = pTerm; |
2883 | pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; |
2884 | |
2885 | assert( nInMul==0 |
2886 | || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 |
2887 | || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 |
2888 | || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 |
2889 | ); |
2890 | |
2891 | if( eOp & WO_IN ){ |
2892 | Expr *pExpr = pTerm->pExpr; |
2893 | if( ExprUseXSelect(pExpr) ){ |
2894 | /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ |
2895 | int i; |
2896 | nIn = 46; assert( 46==sqlite3LogEst(25) ); |
2897 | |
2898 | /* The expression may actually be of the form (x, y) IN (SELECT...). |
2899 | ** In this case there is a separate term for each of (x) and (y). |
2900 | ** However, the nIn multiplier should only be applied once, not once |
2901 | ** for each such term. The following loop checks that pTerm is the |
2902 | ** first such term in use, and sets nIn back to 0 if it is not. */ |
2903 | for(i=0; i<pNew->nLTerm-1; i++){ |
2904 | if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; |
2905 | } |
2906 | }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ |
2907 | /* "x IN (value, value, ...)" */ |
2908 | nIn = sqlite3LogEst(pExpr->x.pList->nExpr); |
2909 | } |
2910 | if( pProbe->hasStat1 && rLogSize>=10 ){ |
2911 | LogEst M, logK, x; |
2912 | /* Let: |
2913 | ** N = the total number of rows in the table |
2914 | ** K = the number of entries on the RHS of the IN operator |
2915 | ** M = the number of rows in the table that match terms to the |
2916 | ** to the left in the same index. If the IN operator is on |
2917 | ** the left-most index column, M==N. |
2918 | ** |
2919 | ** Given the definitions above, it is better to omit the IN operator |
2920 | ** from the index lookup and instead do a scan of the M elements, |
2921 | ** testing each scanned row against the IN operator separately, if: |
2922 | ** |
2923 | ** M*log(K) < K*log(N) |
2924 | ** |
2925 | ** Our estimates for M, K, and N might be inaccurate, so we build in |
2926 | ** a safety margin of 2 (LogEst: 10) that favors using the IN operator |
2927 | ** with the index, as using an index has better worst-case behavior. |
2928 | ** If we do not have real sqlite_stat1 data, always prefer to use |
2929 | ** the index. Do not bother with this optimization on very small |
2930 | ** tables (less than 2 rows) as it is pointless in that case. |
2931 | */ |
2932 | M = pProbe->aiRowLogEst[saved_nEq]; |
2933 | logK = estLog(nIn); |
2934 | /* TUNING v----- 10 to bias toward indexed IN */ |
2935 | x = M + logK + 10 - (nIn + rLogSize); |
2936 | if( x>=0 ){ |
2937 | WHERETRACE(0x40, |
2938 | ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) " |
2939 | "prefers indexed lookup\n" , |
2940 | saved_nEq, M, logK, nIn, rLogSize, x)); |
2941 | }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){ |
2942 | WHERETRACE(0x40, |
2943 | ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d" |
2944 | " nInMul=%d) prefers skip-scan\n" , |
2945 | saved_nEq, M, logK, nIn, rLogSize, x, nInMul)); |
2946 | pNew->wsFlags |= WHERE_IN_SEEKSCAN; |
2947 | }else{ |
2948 | WHERETRACE(0x40, |
2949 | ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d" |
2950 | " nInMul=%d) prefers normal scan\n" , |
2951 | saved_nEq, M, logK, nIn, rLogSize, x, nInMul)); |
2952 | continue; |
2953 | } |
2954 | } |
2955 | pNew->wsFlags |= WHERE_COLUMN_IN; |
2956 | }else if( eOp & (WO_EQ|WO_IS) ){ |
2957 | int iCol = pProbe->aiColumn[saved_nEq]; |
2958 | pNew->wsFlags |= WHERE_COLUMN_EQ; |
2959 | assert( saved_nEq==pNew->u.btree.nEq ); |
2960 | if( iCol==XN_ROWID |
2961 | || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) |
2962 | ){ |
2963 | if( iCol==XN_ROWID || pProbe->uniqNotNull |
2964 | || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) |
2965 | ){ |
2966 | pNew->wsFlags |= WHERE_ONEROW; |
2967 | }else{ |
2968 | pNew->wsFlags |= WHERE_UNQ_WANTED; |
2969 | } |
2970 | } |
2971 | if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS; |
2972 | }else if( eOp & WO_ISNULL ){ |
2973 | pNew->wsFlags |= WHERE_COLUMN_NULL; |
2974 | }else{ |
2975 | int nVecLen = whereRangeVectorLen( |
2976 | pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm |
2977 | ); |
2978 | if( eOp & (WO_GT|WO_GE) ){ |
2979 | testcase( eOp & WO_GT ); |
2980 | testcase( eOp & WO_GE ); |
2981 | pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; |
2982 | pNew->u.btree.nBtm = nVecLen; |
2983 | pBtm = pTerm; |
2984 | pTop = 0; |
2985 | if( pTerm->wtFlags & TERM_LIKEOPT ){ |
2986 | /* Range constraints that come from the LIKE optimization are |
2987 | ** always used in pairs. */ |
2988 | pTop = &pTerm[1]; |
2989 | assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm ); |
2990 | assert( pTop->wtFlags & TERM_LIKEOPT ); |
2991 | assert( pTop->eOperator==WO_LT ); |
2992 | if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ |
2993 | pNew->aLTerm[pNew->nLTerm++] = pTop; |
2994 | pNew->wsFlags |= WHERE_TOP_LIMIT; |
2995 | pNew->u.btree.nTop = 1; |
2996 | } |
2997 | }else{ |
2998 | assert( eOp & (WO_LT|WO_LE) ); |
2999 | testcase( eOp & WO_LT ); |
3000 | testcase( eOp & WO_LE ); |
3001 | pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; |
3002 | pNew->u.btree.nTop = nVecLen; |
3003 | pTop = pTerm; |
3004 | pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? |
3005 | pNew->aLTerm[pNew->nLTerm-2] : 0; |
3006 | } |
3007 | } |
3008 | |
3009 | /* At this point pNew->nOut is set to the number of rows expected to |
3010 | ** be visited by the index scan before considering term pTerm, or the |
3011 | ** values of nIn and nInMul. In other words, assuming that all |
3012 | ** "x IN(...)" terms are replaced with "x = ?". This block updates |
3013 | ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ |
3014 | assert( pNew->nOut==saved_nOut ); |
3015 | if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ |
3016 | /* Adjust nOut using stat4 data. Or, if there is no stat4 |
3017 | ** data, using some other estimate. */ |
3018 | whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); |
3019 | }else{ |
3020 | int nEq = ++pNew->u.btree.nEq; |
3021 | assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); |
3022 | |
3023 | assert( pNew->nOut==saved_nOut ); |
3024 | if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ |
3025 | assert( (eOp & WO_IN) || nIn==0 ); |
3026 | testcase( eOp & WO_IN ); |
3027 | pNew->nOut += pTerm->truthProb; |
3028 | pNew->nOut -= nIn; |
3029 | }else{ |
3030 | #ifdef SQLITE_ENABLE_STAT4 |
3031 | tRowcnt nOut = 0; |
3032 | if( nInMul==0 |
3033 | && pProbe->nSample |
3034 | && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol) |
3035 | && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr)) |
3036 | && OptimizationEnabled(db, SQLITE_Stat4) |
3037 | ){ |
3038 | Expr *pExpr = pTerm->pExpr; |
3039 | if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ |
3040 | testcase( eOp & WO_EQ ); |
3041 | testcase( eOp & WO_IS ); |
3042 | testcase( eOp & WO_ISNULL ); |
3043 | rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); |
3044 | }else{ |
3045 | rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); |
3046 | } |
3047 | if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; |
3048 | if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ |
3049 | if( nOut ){ |
3050 | pNew->nOut = sqlite3LogEst(nOut); |
3051 | if( nEq==1 |
3052 | /* TUNING: Mark terms as "low selectivity" if they seem likely |
3053 | ** to be true for half or more of the rows in the table. |
3054 | ** See tag-202002240-1 */ |
3055 | && pNew->nOut+10 > pProbe->aiRowLogEst[0] |
3056 | ){ |
3057 | #if WHERETRACE_ENABLED /* 0x01 */ |
3058 | if( sqlite3WhereTrace & 0x01 ){ |
3059 | sqlite3DebugPrintf( |
3060 | "STAT4 determines term has low selectivity:\n" ); |
3061 | sqlite3WhereTermPrint(pTerm, 999); |
3062 | } |
3063 | #endif |
3064 | pTerm->wtFlags |= TERM_HIGHTRUTH; |
3065 | if( pTerm->wtFlags & TERM_HEURTRUTH ){ |
3066 | /* If the term has previously been used with an assumption of |
3067 | ** higher selectivity, then set the flag to rerun the |
3068 | ** loop computations. */ |
3069 | pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS; |
3070 | } |
3071 | } |
3072 | if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; |
3073 | pNew->nOut -= nIn; |
3074 | } |
3075 | } |
3076 | if( nOut==0 ) |
3077 | #endif |
3078 | { |
3079 | pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); |
3080 | if( eOp & WO_ISNULL ){ |
3081 | /* TUNING: If there is no likelihood() value, assume that a |
3082 | ** "col IS NULL" expression matches twice as many rows |
3083 | ** as (col=?). */ |
3084 | pNew->nOut += 10; |
3085 | } |
3086 | } |
3087 | } |
3088 | } |
3089 | |
3090 | /* Set rCostIdx to the cost of visiting selected rows in index. Add |
3091 | ** it to pNew->rRun, which is currently set to the cost of the index |
3092 | ** seek only. Then, if this is a non-covering index, add the cost of |
3093 | ** visiting the rows in the main table. */ |
3094 | assert( pSrc->pTab->szTabRow>0 ); |
3095 | rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; |
3096 | pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); |
3097 | if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ |
3098 | pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); |
3099 | } |
3100 | ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); |
3101 | |
3102 | nOutUnadjusted = pNew->nOut; |
3103 | pNew->rRun += nInMul + nIn; |
3104 | pNew->nOut += nInMul + nIn; |
3105 | whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); |
3106 | rc = whereLoopInsert(pBuilder, pNew); |
3107 | |
3108 | if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ |
3109 | pNew->nOut = saved_nOut; |
3110 | }else{ |
3111 | pNew->nOut = nOutUnadjusted; |
3112 | } |
3113 | |
3114 | if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 |
3115 | && pNew->u.btree.nEq<pProbe->nColumn |
3116 | && (pNew->u.btree.nEq<pProbe->nKeyCol || |
3117 | pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) |
3118 | ){ |
3119 | whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); |
3120 | } |
3121 | pNew->nOut = saved_nOut; |
3122 | #ifdef SQLITE_ENABLE_STAT4 |
3123 | pBuilder->nRecValid = nRecValid; |
3124 | #endif |
3125 | } |
3126 | pNew->prereq = saved_prereq; |
3127 | pNew->u.btree.nEq = saved_nEq; |
3128 | pNew->u.btree.nBtm = saved_nBtm; |
3129 | pNew->u.btree.nTop = saved_nTop; |
3130 | pNew->nSkip = saved_nSkip; |
3131 | pNew->wsFlags = saved_wsFlags; |
3132 | pNew->nOut = saved_nOut; |
3133 | pNew->nLTerm = saved_nLTerm; |
3134 | |
3135 | /* Consider using a skip-scan if there are no WHERE clause constraints |
3136 | ** available for the left-most terms of the index, and if the average |
3137 | ** number of repeats in the left-most terms is at least 18. |
3138 | ** |
3139 | ** The magic number 18 is selected on the basis that scanning 17 rows |
3140 | ** is almost always quicker than an index seek (even though if the index |
3141 | ** contains fewer than 2^17 rows we assume otherwise in other parts of |
3142 | ** the code). And, even if it is not, it should not be too much slower. |
3143 | ** On the other hand, the extra seeks could end up being significantly |
3144 | ** more expensive. */ |
3145 | assert( 42==sqlite3LogEst(18) ); |
3146 | if( saved_nEq==saved_nSkip |
3147 | && saved_nEq+1<pProbe->nKeyCol |
3148 | && saved_nEq==pNew->nLTerm |
3149 | && pProbe->noSkipScan==0 |
3150 | && pProbe->hasStat1!=0 |
3151 | && OptimizationEnabled(db, SQLITE_SkipScan) |
3152 | && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ |
3153 | && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK |
3154 | ){ |
3155 | LogEst nIter; |
3156 | pNew->u.btree.nEq++; |
3157 | pNew->nSkip++; |
3158 | pNew->aLTerm[pNew->nLTerm++] = 0; |
3159 | pNew->wsFlags |= WHERE_SKIPSCAN; |
3160 | nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; |
3161 | pNew->nOut -= nIter; |
3162 | /* TUNING: Because uncertainties in the estimates for skip-scan queries, |
3163 | ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ |
3164 | nIter += 5; |
3165 | whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); |
3166 | pNew->nOut = saved_nOut; |
3167 | pNew->u.btree.nEq = saved_nEq; |
3168 | pNew->nSkip = saved_nSkip; |
3169 | pNew->wsFlags = saved_wsFlags; |
3170 | } |
3171 | |
3172 | WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n" , |
3173 | pProbe->pTable->zName, pProbe->zName, saved_nEq, rc)); |
3174 | return rc; |
3175 | } |
3176 | |
3177 | /* |
3178 | ** Return True if it is possible that pIndex might be useful in |
3179 | ** implementing the ORDER BY clause in pBuilder. |
3180 | ** |
3181 | ** Return False if pBuilder does not contain an ORDER BY clause or |
3182 | ** if there is no way for pIndex to be useful in implementing that |
3183 | ** ORDER BY clause. |
3184 | */ |
3185 | static int indexMightHelpWithOrderBy( |
3186 | WhereLoopBuilder *pBuilder, |
3187 | Index *pIndex, |
3188 | int iCursor |
3189 | ){ |
3190 | ExprList *pOB; |
3191 | ExprList *aColExpr; |
3192 | int ii, jj; |
3193 | |
3194 | if( pIndex->bUnordered ) return 0; |
3195 | if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; |
3196 | for(ii=0; ii<pOB->nExpr; ii++){ |
3197 | Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); |
3198 | if( NEVER(pExpr==0) ) continue; |
3199 | if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ |
3200 | if( pExpr->iColumn<0 ) return 1; |
3201 | for(jj=0; jj<pIndex->nKeyCol; jj++){ |
3202 | if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; |
3203 | } |
3204 | }else if( (aColExpr = pIndex->aColExpr)!=0 ){ |
3205 | for(jj=0; jj<pIndex->nKeyCol; jj++){ |
3206 | if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; |
3207 | if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ |
3208 | return 1; |
3209 | } |
3210 | } |
3211 | } |
3212 | } |
3213 | return 0; |
3214 | } |
3215 | |
3216 | /* Check to see if a partial index with pPartIndexWhere can be used |
3217 | ** in the current query. Return true if it can be and false if not. |
3218 | */ |
3219 | static int whereUsablePartialIndex( |
3220 | int iTab, /* The table for which we want an index */ |
3221 | u8 jointype, /* The JT_* flags on the join */ |
3222 | WhereClause *pWC, /* The WHERE clause of the query */ |
3223 | Expr *pWhere /* The WHERE clause from the partial index */ |
3224 | ){ |
3225 | int i; |
3226 | WhereTerm *pTerm; |
3227 | Parse *pParse; |
3228 | |
3229 | if( jointype & JT_LTORJ ) return 0; |
3230 | pParse = pWC->pWInfo->pParse; |
3231 | while( pWhere->op==TK_AND ){ |
3232 | if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0; |
3233 | pWhere = pWhere->pRight; |
3234 | } |
3235 | if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0; |
3236 | for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){ |
3237 | Expr *pExpr; |
3238 | pExpr = pTerm->pExpr; |
3239 | if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab) |
3240 | && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON)) |
3241 | && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) |
3242 | && (pTerm->wtFlags & TERM_VNULL)==0 |
3243 | ){ |
3244 | return 1; |
3245 | } |
3246 | } |
3247 | return 0; |
3248 | } |
3249 | |
3250 | /* |
3251 | ** Structure passed to the whereIsCoveringIndex Walker callback. |
3252 | */ |
3253 | struct CoveringIndexCheck { |
3254 | Index *pIdx; /* The index */ |
3255 | int iTabCur; /* Cursor number for the corresponding table */ |
3256 | }; |
3257 | |
3258 | /* |
3259 | ** Information passed in is pWalk->u.pCovIdxCk. Call is pCk. |
3260 | ** |
3261 | ** If the Expr node references the table with cursor pCk->iTabCur, then |
3262 | ** make sure that column is covered by the index pCk->pIdx. We know that |
3263 | ** all columns less than 63 (really BMS-1) are covered, so we don't need |
3264 | ** to check them. But we do need to check any column at 63 or greater. |
3265 | ** |
3266 | ** If the index does not cover the column, then set pWalk->eCode to |
3267 | ** non-zero and return WRC_Abort to stop the search. |
3268 | ** |
3269 | ** If this node does not disprove that the index can be a covering index, |
3270 | ** then just return WRC_Continue, to continue the search. |
3271 | */ |
3272 | static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ |
3273 | int i; /* Loop counter */ |
3274 | const Index *pIdx; /* The index of interest */ |
3275 | const i16 *aiColumn; /* Columns contained in the index */ |
3276 | u16 nColumn; /* Number of columns in the index */ |
3277 | if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue; |
3278 | if( pExpr->iColumn<(BMS-1) ) return WRC_Continue; |
3279 | if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue; |
3280 | pIdx = pWalk->u.pCovIdxCk->pIdx; |
3281 | aiColumn = pIdx->aiColumn; |
3282 | nColumn = pIdx->nColumn; |
3283 | for(i=0; i<nColumn; i++){ |
3284 | if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue; |
3285 | } |
3286 | pWalk->eCode = 1; |
3287 | return WRC_Abort; |
3288 | } |
3289 | |
3290 | |
3291 | /* |
3292 | ** pIdx is an index that covers all of the low-number columns used by |
3293 | ** pWInfo->pSelect (columns from 0 through 62). But there are columns |
3294 | ** in pWInfo->pSelect beyond 62. This routine tries to answer the question |
3295 | ** of whether pIdx covers *all* columns in the query. |
3296 | ** |
3297 | ** Return 0 if pIdx is a covering index. Return non-zero if pIdx is |
3298 | ** not a covering index or if we are unable to determine if pIdx is a |
3299 | ** covering index. |
3300 | ** |
3301 | ** This routine is an optimization. It is always safe to return non-zero. |
3302 | ** But returning zero when non-zero should have been returned can lead to |
3303 | ** incorrect bytecode and assertion faults. |
3304 | */ |
3305 | static SQLITE_NOINLINE u32 whereIsCoveringIndex( |
3306 | WhereInfo *pWInfo, /* The WHERE clause context */ |
3307 | Index *pIdx, /* Index that is being tested */ |
3308 | int iTabCur /* Cursor for the table being indexed */ |
3309 | ){ |
3310 | int i; |
3311 | struct CoveringIndexCheck ck; |
3312 | Walker w; |
3313 | if( pWInfo->pSelect==0 ){ |
3314 | /* We don't have access to the full query, so we cannot check to see |
3315 | ** if pIdx is covering. Assume it is not. */ |
3316 | return 1; |
3317 | } |
3318 | for(i=0; i<pIdx->nColumn; i++){ |
3319 | if( pIdx->aiColumn[i]>=BMS-1 ) break; |
3320 | } |
3321 | if( i>=pIdx->nColumn ){ |
3322 | /* pIdx does not index any columns greater than 62, but we know from |
3323 | ** colMask that columns greater than 62 are used, so this is not a |
3324 | ** covering index */ |
3325 | return 1; |
3326 | } |
3327 | ck.pIdx = pIdx; |
3328 | ck.iTabCur = iTabCur; |
3329 | memset(&w, 0, sizeof(w)); |
3330 | w.xExprCallback = whereIsCoveringIndexWalkCallback; |
3331 | w.xSelectCallback = sqlite3SelectWalkNoop; |
3332 | w.u.pCovIdxCk = &ck; |
3333 | w.eCode = 0; |
3334 | sqlite3WalkSelect(&w, pWInfo->pSelect); |
3335 | return w.eCode; |
3336 | } |
3337 | |
3338 | /* |
3339 | ** Add all WhereLoop objects for a single table of the join where the table |
3340 | ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be |
3341 | ** a b-tree table, not a virtual table. |
3342 | ** |
3343 | ** The costs (WhereLoop.rRun) of the b-tree loops added by this function |
3344 | ** are calculated as follows: |
3345 | ** |
3346 | ** For a full scan, assuming the table (or index) contains nRow rows: |
3347 | ** |
3348 | ** cost = nRow * 3.0 // full-table scan |
3349 | ** cost = nRow * K // scan of covering index |
3350 | ** cost = nRow * (K+3.0) // scan of non-covering index |
3351 | ** |
3352 | ** where K is a value between 1.1 and 3.0 set based on the relative |
3353 | ** estimated average size of the index and table records. |
3354 | ** |
3355 | ** For an index scan, where nVisit is the number of index rows visited |
3356 | ** by the scan, and nSeek is the number of seek operations required on |
3357 | ** the index b-tree: |
3358 | ** |
3359 | ** cost = nSeek * (log(nRow) + K * nVisit) // covering index |
3360 | ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index |
3361 | ** |
3362 | ** Normally, nSeek is 1. nSeek values greater than 1 come about if the |
3363 | ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when |
3364 | ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. |
3365 | ** |
3366 | ** The estimated values (nRow, nVisit, nSeek) often contain a large amount |
3367 | ** of uncertainty. For this reason, scoring is designed to pick plans that |
3368 | ** "do the least harm" if the estimates are inaccurate. For example, a |
3369 | ** log(nRow) factor is omitted from a non-covering index scan in order to |
3370 | ** bias the scoring in favor of using an index, since the worst-case |
3371 | ** performance of using an index is far better than the worst-case performance |
3372 | ** of a full table scan. |
3373 | */ |
3374 | static int whereLoopAddBtree( |
3375 | WhereLoopBuilder *pBuilder, /* WHERE clause information */ |
3376 | Bitmask mPrereq /* Extra prerequesites for using this table */ |
3377 | ){ |
3378 | WhereInfo *pWInfo; /* WHERE analysis context */ |
3379 | Index *pProbe; /* An index we are evaluating */ |
3380 | Index sPk; /* A fake index object for the primary key */ |
3381 | LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ |
3382 | i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ |
3383 | SrcList *pTabList; /* The FROM clause */ |
3384 | SrcItem *pSrc; /* The FROM clause btree term to add */ |
3385 | WhereLoop *pNew; /* Template WhereLoop object */ |
3386 | int rc = SQLITE_OK; /* Return code */ |
3387 | int iSortIdx = 1; /* Index number */ |
3388 | int b; /* A boolean value */ |
3389 | LogEst rSize; /* number of rows in the table */ |
3390 | WhereClause *pWC; /* The parsed WHERE clause */ |
3391 | Table *pTab; /* Table being queried */ |
3392 | |
3393 | pNew = pBuilder->pNew; |
3394 | pWInfo = pBuilder->pWInfo; |
3395 | pTabList = pWInfo->pTabList; |
3396 | pSrc = pTabList->a + pNew->iTab; |
3397 | pTab = pSrc->pTab; |
3398 | pWC = pBuilder->pWC; |
3399 | assert( !IsVirtual(pSrc->pTab) ); |
3400 | |
3401 | if( pSrc->fg.isIndexedBy ){ |
3402 | assert( pSrc->fg.isCte==0 ); |
3403 | /* An INDEXED BY clause specifies a particular index to use */ |
3404 | pProbe = pSrc->u2.pIBIndex; |
3405 | }else if( !HasRowid(pTab) ){ |
3406 | pProbe = pTab->pIndex; |
3407 | }else{ |
3408 | /* There is no INDEXED BY clause. Create a fake Index object in local |
3409 | ** variable sPk to represent the rowid primary key index. Make this |
3410 | ** fake index the first in a chain of Index objects with all of the real |
3411 | ** indices to follow */ |
3412 | Index *pFirst; /* First of real indices on the table */ |
3413 | memset(&sPk, 0, sizeof(Index)); |
3414 | sPk.nKeyCol = 1; |
3415 | sPk.nColumn = 1; |
3416 | sPk.aiColumn = &aiColumnPk; |
3417 | sPk.aiRowLogEst = aiRowEstPk; |
3418 | sPk.onError = OE_Replace; |
3419 | sPk.pTable = pTab; |
3420 | sPk.szIdxRow = pTab->szTabRow; |
3421 | sPk.idxType = SQLITE_IDXTYPE_IPK; |
3422 | aiRowEstPk[0] = pTab->nRowLogEst; |
3423 | aiRowEstPk[1] = 0; |
3424 | pFirst = pSrc->pTab->pIndex; |
3425 | if( pSrc->fg.notIndexed==0 ){ |
3426 | /* The real indices of the table are only considered if the |
3427 | ** NOT INDEXED qualifier is omitted from the FROM clause */ |
3428 | sPk.pNext = pFirst; |
3429 | } |
3430 | pProbe = &sPk; |
3431 | } |
3432 | rSize = pTab->nRowLogEst; |
3433 | |
3434 | #ifndef SQLITE_OMIT_AUTOMATIC_INDEX |
3435 | /* Automatic indexes */ |
3436 | if( !pBuilder->pOrSet /* Not part of an OR optimization */ |
3437 | && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0 |
3438 | && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 |
3439 | && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */ |
3440 | && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ |
3441 | && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ |
3442 | && !pSrc->fg.isCorrelated /* Not a correlated subquery */ |
3443 | && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ |
3444 | && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */ |
3445 | ){ |
3446 | /* Generate auto-index WhereLoops */ |
3447 | LogEst rLogSize; /* Logarithm of the number of rows in the table */ |
3448 | WhereTerm *pTerm; |
3449 | WhereTerm *pWCEnd = pWC->a + pWC->nTerm; |
3450 | rLogSize = estLog(rSize); |
3451 | for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){ |
3452 | if( pTerm->prereqRight & pNew->maskSelf ) continue; |
3453 | if( termCanDriveIndex(pTerm, pSrc, 0) ){ |
3454 | pNew->u.btree.nEq = 1; |
3455 | pNew->nSkip = 0; |
3456 | pNew->u.btree.pIndex = 0; |
3457 | pNew->nLTerm = 1; |
3458 | pNew->aLTerm[0] = pTerm; |
3459 | /* TUNING: One-time cost for computing the automatic index is |
3460 | ** estimated to be X*N*log2(N) where N is the number of rows in |
3461 | ** the table being indexed and where X is 7 (LogEst=28) for normal |
3462 | ** tables or 0.5 (LogEst=-10) for views and subqueries. The value |
3463 | ** of X is smaller for views and subqueries so that the query planner |
3464 | ** will be more aggressive about generating automatic indexes for |
3465 | ** those objects, since there is no opportunity to add schema |
3466 | ** indexes on subqueries and views. */ |
3467 | pNew->rSetup = rLogSize + rSize; |
3468 | if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){ |
3469 | pNew->rSetup += 28; |
3470 | }else{ |
3471 | pNew->rSetup -= 10; |
3472 | } |
3473 | ApplyCostMultiplier(pNew->rSetup, pTab->costMult); |
3474 | if( pNew->rSetup<0 ) pNew->rSetup = 0; |
3475 | /* TUNING: Each index lookup yields 20 rows in the table. This |
3476 | ** is more than the usual guess of 10 rows, since we have no way |
3477 | ** of knowing how selective the index will ultimately be. It would |
3478 | ** not be unreasonable to make this value much larger. */ |
3479 | pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); |
3480 | pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); |
3481 | pNew->wsFlags = WHERE_AUTO_INDEX; |
3482 | pNew->prereq = mPrereq | pTerm->prereqRight; |
3483 | rc = whereLoopInsert(pBuilder, pNew); |
3484 | } |
3485 | } |
3486 | } |
3487 | #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ |
3488 | |
3489 | /* Loop over all indices. If there was an INDEXED BY clause, then only |
3490 | ** consider index pProbe. */ |
3491 | for(; rc==SQLITE_OK && pProbe; |
3492 | pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++ |
3493 | ){ |
3494 | if( pProbe->pPartIdxWhere!=0 |
3495 | && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC, |
3496 | pProbe->pPartIdxWhere) |
3497 | ){ |
3498 | testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ |
3499 | continue; /* Partial index inappropriate for this query */ |
3500 | } |
3501 | if( pProbe->bNoQuery ) continue; |
3502 | rSize = pProbe->aiRowLogEst[0]; |
3503 | pNew->u.btree.nEq = 0; |
3504 | pNew->u.btree.nBtm = 0; |
3505 | pNew->u.btree.nTop = 0; |
3506 | pNew->nSkip = 0; |
3507 | pNew->nLTerm = 0; |
3508 | pNew->iSortIdx = 0; |
3509 | pNew->rSetup = 0; |
3510 | pNew->prereq = mPrereq; |
3511 | pNew->nOut = rSize; |
3512 | pNew->u.btree.pIndex = pProbe; |
3513 | b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); |
3514 | |
3515 | /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ |
3516 | assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); |
3517 | if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ |
3518 | /* Integer primary key index */ |
3519 | pNew->wsFlags = WHERE_IPK; |
3520 | |
3521 | /* Full table scan */ |
3522 | pNew->iSortIdx = b ? iSortIdx : 0; |
3523 | /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an |
3524 | ** extra cost designed to discourage the use of full table scans, |
3525 | ** since index lookups have better worst-case performance if our |
3526 | ** stat guesses are wrong. Reduce the 3.0 penalty slightly |
3527 | ** (to 2.75) if we have valid STAT4 information for the table. |
3528 | ** At 2.75, a full table scan is preferred over using an index on |
3529 | ** a column with just two distinct values where each value has about |
3530 | ** an equal number of appearances. Without STAT4 data, we still want |
3531 | ** to use an index in that case, since the constraint might be for |
3532 | ** the scarcer of the two values, and in that case an index lookup is |
3533 | ** better. |
3534 | */ |
3535 | #ifdef SQLITE_ENABLE_STAT4 |
3536 | pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0); |
3537 | #else |
3538 | pNew->rRun = rSize + 16; |
3539 | #endif |
3540 | if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){ |
3541 | pNew->wsFlags |= WHERE_VIEWSCAN; |
3542 | } |
3543 | ApplyCostMultiplier(pNew->rRun, pTab->costMult); |
3544 | whereLoopOutputAdjust(pWC, pNew, rSize); |
3545 | rc = whereLoopInsert(pBuilder, pNew); |
3546 | pNew->nOut = rSize; |
3547 | if( rc ) break; |
3548 | }else{ |
3549 | Bitmask m; |
3550 | if( pProbe->isCovering ){ |
3551 | pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; |
3552 | m = 0; |
3553 | }else{ |
3554 | m = pSrc->colUsed & pProbe->colNotIdxed; |
3555 | if( m==TOPBIT ){ |
3556 | m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); |
3557 | } |
3558 | pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; |
3559 | } |
3560 | |
3561 | /* Full scan via index */ |
3562 | if( b |
3563 | || !HasRowid(pTab) |
3564 | || pProbe->pPartIdxWhere!=0 |
3565 | || pSrc->fg.isIndexedBy |
3566 | || ( m==0 |
3567 | && pProbe->bUnordered==0 |
3568 | && (pProbe->szIdxRow<pTab->szTabRow) |
3569 | && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 |
3570 | && sqlite3GlobalConfig.bUseCis |
3571 | && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) |
3572 | ) |
3573 | ){ |
3574 | pNew->iSortIdx = b ? iSortIdx : 0; |
3575 | |
3576 | /* The cost of visiting the index rows is N*K, where K is |
3577 | ** between 1.1 and 3.0, depending on the relative sizes of the |
3578 | ** index and table rows. */ |
3579 | pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; |
3580 | if( m!=0 ){ |
3581 | /* If this is a non-covering index scan, add in the cost of |
3582 | ** doing table lookups. The cost will be 3x the number of |
3583 | ** lookups. Take into account WHERE clause terms that can be |
3584 | ** satisfied using just the index, and that do not require a |
3585 | ** table lookup. */ |
3586 | LogEst nLookup = rSize + 16; /* Base cost: N*3 */ |
3587 | int ii; |
3588 | int iCur = pSrc->iCursor; |
3589 | WhereClause *pWC2 = &pWInfo->sWC; |
3590 | for(ii=0; ii<pWC2->nTerm; ii++){ |
3591 | WhereTerm *pTerm = &pWC2->a[ii]; |
3592 | if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){ |
3593 | break; |
3594 | } |
3595 | /* pTerm can be evaluated using just the index. So reduce |
3596 | ** the expected number of table lookups accordingly */ |
3597 | if( pTerm->truthProb<=0 ){ |
3598 | nLookup += pTerm->truthProb; |
3599 | }else{ |
3600 | nLookup--; |
3601 | if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19; |
3602 | } |
3603 | } |
3604 | |
3605 | pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup); |
3606 | } |
3607 | ApplyCostMultiplier(pNew->rRun, pTab->costMult); |
3608 | whereLoopOutputAdjust(pWC, pNew, rSize); |
3609 | if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){ |
3610 | /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN |
3611 | ** because the cursor used to access the index might not be |
3612 | ** positioned to the correct row during the right-join no-match |
3613 | ** loop. */ |
3614 | }else{ |
3615 | rc = whereLoopInsert(pBuilder, pNew); |
3616 | } |
3617 | pNew->nOut = rSize; |
3618 | if( rc ) break; |
3619 | } |
3620 | } |
3621 | |
3622 | pBuilder->bldFlags1 = 0; |
3623 | rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); |
3624 | if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){ |
3625 | /* If a non-unique index is used, or if a prefix of the key for |
3626 | ** unique index is used (making the index functionally non-unique) |
3627 | ** then the sqlite_stat1 data becomes important for scoring the |
3628 | ** plan */ |
3629 | pTab->tabFlags |= TF_StatsUsed; |
3630 | } |
3631 | #ifdef SQLITE_ENABLE_STAT4 |
3632 | sqlite3Stat4ProbeFree(pBuilder->pRec); |
3633 | pBuilder->nRecValid = 0; |
3634 | pBuilder->pRec = 0; |
3635 | #endif |
3636 | } |
3637 | return rc; |
3638 | } |
3639 | |
3640 | #ifndef SQLITE_OMIT_VIRTUALTABLE |
3641 | |
3642 | /* |
3643 | ** Return true if pTerm is a virtual table LIMIT or OFFSET term. |
3644 | */ |
3645 | static int isLimitTerm(WhereTerm *pTerm){ |
3646 | assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 ); |
3647 | return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT |
3648 | && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET; |
3649 | } |
3650 | |
3651 | /* |
3652 | ** Argument pIdxInfo is already populated with all constraints that may |
3653 | ** be used by the virtual table identified by pBuilder->pNew->iTab. This |
3654 | ** function marks a subset of those constraints usable, invokes the |
3655 | ** xBestIndex method and adds the returned plan to pBuilder. |
3656 | ** |
3657 | ** A constraint is marked usable if: |
3658 | ** |
3659 | ** * Argument mUsable indicates that its prerequisites are available, and |
3660 | ** |
3661 | ** * It is not one of the operators specified in the mExclude mask passed |
3662 | ** as the fourth argument (which in practice is either WO_IN or 0). |
3663 | ** |
3664 | ** Argument mPrereq is a mask of tables that must be scanned before the |
3665 | ** virtual table in question. These are added to the plans prerequisites |
3666 | ** before it is added to pBuilder. |
3667 | ** |
3668 | ** Output parameter *pbIn is set to true if the plan added to pBuilder |
3669 | ** uses one or more WO_IN terms, or false otherwise. |
3670 | */ |
3671 | static int whereLoopAddVirtualOne( |
3672 | WhereLoopBuilder *pBuilder, |
3673 | Bitmask mPrereq, /* Mask of tables that must be used. */ |
3674 | Bitmask mUsable, /* Mask of usable tables */ |
3675 | u16 mExclude, /* Exclude terms using these operators */ |
3676 | sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */ |
3677 | u16 mNoOmit, /* Do not omit these constraints */ |
3678 | int *pbIn, /* OUT: True if plan uses an IN(...) op */ |
3679 | int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */ |
3680 | ){ |
3681 | WhereClause *pWC = pBuilder->pWC; |
3682 | HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; |
3683 | struct sqlite3_index_constraint *pIdxCons; |
3684 | struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage; |
3685 | int i; |
3686 | int mxTerm; |
3687 | int rc = SQLITE_OK; |
3688 | WhereLoop *pNew = pBuilder->pNew; |
3689 | Parse *pParse = pBuilder->pWInfo->pParse; |
3690 | SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab]; |
3691 | int nConstraint = pIdxInfo->nConstraint; |
3692 | |
3693 | assert( (mUsable & mPrereq)==mPrereq ); |
3694 | *pbIn = 0; |
3695 | pNew->prereq = mPrereq; |
3696 | |
3697 | /* Set the usable flag on the subset of constraints identified by |
3698 | ** arguments mUsable and mExclude. */ |
3699 | pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; |
3700 | for(i=0; i<nConstraint; i++, pIdxCons++){ |
3701 | WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset]; |
3702 | pIdxCons->usable = 0; |
3703 | if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight |
3704 | && (pTerm->eOperator & mExclude)==0 |
3705 | && (pbRetryLimit || !isLimitTerm(pTerm)) |
3706 | ){ |
3707 | pIdxCons->usable = 1; |
3708 | } |
3709 | } |
3710 | |
3711 | /* Initialize the output fields of the sqlite3_index_info structure */ |
3712 | memset(pUsage, 0, sizeof(pUsage[0])*nConstraint); |
3713 | assert( pIdxInfo->needToFreeIdxStr==0 ); |
3714 | pIdxInfo->idxStr = 0; |
3715 | pIdxInfo->idxNum = 0; |
3716 | pIdxInfo->orderByConsumed = 0; |
3717 | pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; |
3718 | pIdxInfo->estimatedRows = 25; |
3719 | pIdxInfo->idxFlags = 0; |
3720 | pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; |
3721 | pHidden->mHandleIn = 0; |
3722 | |
3723 | /* Invoke the virtual table xBestIndex() method */ |
3724 | rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); |
3725 | if( rc ){ |
3726 | if( rc==SQLITE_CONSTRAINT ){ |
3727 | /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means |
3728 | ** that the particular combination of parameters provided is unusable. |
3729 | ** Make no entries in the loop table. |
3730 | */ |
3731 | WHERETRACE(0xffff, (" ^^^^--- non-viable plan rejected!\n" )); |
3732 | return SQLITE_OK; |
3733 | } |
3734 | return rc; |
3735 | } |
3736 | |
3737 | mxTerm = -1; |
3738 | assert( pNew->nLSlot>=nConstraint ); |
3739 | memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint ); |
3740 | memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab)); |
3741 | pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; |
3742 | for(i=0; i<nConstraint; i++, pIdxCons++){ |
3743 | int iTerm; |
3744 | if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){ |
3745 | WhereTerm *pTerm; |
3746 | int j = pIdxCons->iTermOffset; |
3747 | if( iTerm>=nConstraint |
3748 | || j<0 |
3749 | || j>=pWC->nTerm |
3750 | || pNew->aLTerm[iTerm]!=0 |
3751 | || pIdxCons->usable==0 |
3752 | ){ |
3753 | sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction" ,pSrc->pTab->zName); |
3754 | testcase( pIdxInfo->needToFreeIdxStr ); |
3755 | return SQLITE_ERROR; |
3756 | } |
3757 | testcase( iTerm==nConstraint-1 ); |
3758 | testcase( j==0 ); |
3759 | testcase( j==pWC->nTerm-1 ); |
3760 | pTerm = &pWC->a[j]; |
3761 | pNew->prereq |= pTerm->prereqRight; |
3762 | assert( iTerm<pNew->nLSlot ); |
3763 | pNew->aLTerm[iTerm] = pTerm; |
3764 | if( iTerm>mxTerm ) mxTerm = iTerm; |
3765 | testcase( iTerm==15 ); |
3766 | testcase( iTerm==16 ); |
3767 | if( pUsage[i].omit ){ |
3768 | if( i<16 && ((1<<i)&mNoOmit)==0 ){ |
3769 | testcase( i!=iTerm ); |
3770 | pNew->u.vtab.omitMask |= 1<<iTerm; |
3771 | }else{ |
3772 | testcase( i!=iTerm ); |
3773 | } |
3774 | if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){ |
3775 | pNew->u.vtab.bOmitOffset = 1; |
3776 | } |
3777 | } |
3778 | if( SMASKBIT32(i) & pHidden->mHandleIn ){ |
3779 | pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm); |
3780 | }else if( (pTerm->eOperator & WO_IN)!=0 ){ |
3781 | /* A virtual table that is constrained by an IN clause may not |
3782 | ** consume the ORDER BY clause because (1) the order of IN terms |
3783 | ** is not necessarily related to the order of output terms and |
3784 | ** (2) Multiple outputs from a single IN value will not merge |
3785 | ** together. */ |
3786 | pIdxInfo->orderByConsumed = 0; |
3787 | pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; |
3788 | *pbIn = 1; assert( (mExclude & WO_IN)==0 ); |
3789 | } |
3790 | |
3791 | assert( pbRetryLimit || !isLimitTerm(pTerm) ); |
3792 | if( isLimitTerm(pTerm) && *pbIn ){ |
3793 | /* If there is an IN(...) term handled as an == (separate call to |
3794 | ** xFilter for each value on the RHS of the IN) and a LIMIT or |
3795 | ** OFFSET term handled as well, the plan is unusable. Set output |
3796 | ** variable *pbRetryLimit to true to tell the caller to retry with |
3797 | ** LIMIT and OFFSET disabled. */ |
3798 | if( pIdxInfo->needToFreeIdxStr ){ |
3799 | sqlite3_free(pIdxInfo->idxStr); |
3800 | pIdxInfo->idxStr = 0; |
3801 | pIdxInfo->needToFreeIdxStr = 0; |
3802 | } |
3803 | *pbRetryLimit = 1; |
3804 | return SQLITE_OK; |
3805 | } |
3806 | } |
3807 | } |
3808 | |
3809 | pNew->nLTerm = mxTerm+1; |
3810 | for(i=0; i<=mxTerm; i++){ |
3811 | if( pNew->aLTerm[i]==0 ){ |
3812 | /* The non-zero argvIdx values must be contiguous. Raise an |
3813 | ** error if they are not */ |
3814 | sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction" ,pSrc->pTab->zName); |
3815 | testcase( pIdxInfo->needToFreeIdxStr ); |
3816 | return SQLITE_ERROR; |
3817 | } |
3818 | } |
3819 | assert( pNew->nLTerm<=pNew->nLSlot ); |
3820 | pNew->u.vtab.idxNum = pIdxInfo->idxNum; |
3821 | pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; |
3822 | pIdxInfo->needToFreeIdxStr = 0; |
3823 | pNew->u.vtab.idxStr = pIdxInfo->idxStr; |
3824 | pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? |
3825 | pIdxInfo->nOrderBy : 0); |
3826 | pNew->rSetup = 0; |
3827 | pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); |
3828 | pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); |
3829 | |
3830 | /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated |
3831 | ** that the scan will visit at most one row. Clear it otherwise. */ |
3832 | if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ |
3833 | pNew->wsFlags |= WHERE_ONEROW; |
3834 | }else{ |
3835 | pNew->wsFlags &= ~WHERE_ONEROW; |
3836 | } |
3837 | rc = whereLoopInsert(pBuilder, pNew); |
3838 | if( pNew->u.vtab.needFree ){ |
3839 | sqlite3_free(pNew->u.vtab.idxStr); |
3840 | pNew->u.vtab.needFree = 0; |
3841 | } |
3842 | WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n" , |
3843 | *pbIn, (sqlite3_uint64)mPrereq, |
3844 | (sqlite3_uint64)(pNew->prereq & ~mPrereq))); |
3845 | |
3846 | return rc; |
3847 | } |
3848 | |
3849 | /* |
3850 | ** Return the collating sequence for a constraint passed into xBestIndex. |
3851 | ** |
3852 | ** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex. |
3853 | ** This routine depends on there being a HiddenIndexInfo structure immediately |
3854 | ** following the sqlite3_index_info structure. |
3855 | ** |
3856 | ** Return a pointer to the collation name: |
3857 | ** |
3858 | ** 1. If there is an explicit COLLATE operator on the constaint, return it. |
3859 | ** |
3860 | ** 2. Else, if the column has an alternative collation, return that. |
3861 | ** |
3862 | ** 3. Otherwise, return "BINARY". |
3863 | */ |
3864 | const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){ |
3865 | HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; |
3866 | const char *zRet = 0; |
3867 | if( iCons>=0 && iCons<pIdxInfo->nConstraint ){ |
3868 | CollSeq *pC = 0; |
3869 | int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset; |
3870 | Expr *pX = pHidden->pWC->a[iTerm].pExpr; |
3871 | if( pX->pLeft ){ |
3872 | pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX); |
3873 | } |
3874 | zRet = (pC ? pC->zName : sqlite3StrBINARY); |
3875 | } |
3876 | return zRet; |
3877 | } |
3878 | |
3879 | /* |
3880 | ** Return true if constraint iCons is really an IN(...) constraint, or |
3881 | ** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0) |
3882 | ** or clear (if bHandle==0) the flag to handle it using an iterator. |
3883 | */ |
3884 | int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){ |
3885 | HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; |
3886 | u32 m = SMASKBIT32(iCons); |
3887 | if( m & pHidden->mIn ){ |
3888 | if( bHandle==0 ){ |
3889 | pHidden->mHandleIn &= ~m; |
3890 | }else if( bHandle>0 ){ |
3891 | pHidden->mHandleIn |= m; |
3892 | } |
3893 | return 1; |
3894 | } |
3895 | return 0; |
3896 | } |
3897 | |
3898 | /* |
3899 | ** This interface is callable from within the xBestIndex callback only. |
3900 | ** |
3901 | ** If possible, set (*ppVal) to point to an object containing the value |
3902 | ** on the right-hand-side of constraint iCons. |
3903 | */ |
3904 | int sqlite3_vtab_rhs_value( |
3905 | sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */ |
3906 | int iCons, /* Constraint for which RHS is wanted */ |
3907 | sqlite3_value **ppVal /* Write value extracted here */ |
3908 | ){ |
3909 | HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1]; |
3910 | sqlite3_value *pVal = 0; |
3911 | int rc = SQLITE_OK; |
3912 | if( iCons<0 || iCons>=pIdxInfo->nConstraint ){ |
3913 | rc = SQLITE_MISUSE; /* EV: R-30545-25046 */ |
3914 | }else{ |
3915 | if( pH->aRhs[iCons]==0 ){ |
3916 | WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset]; |
3917 | rc = sqlite3ValueFromExpr( |
3918 | pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db), |
3919 | SQLITE_AFF_BLOB, &pH->aRhs[iCons] |
3920 | ); |
3921 | testcase( rc!=SQLITE_OK ); |
3922 | } |
3923 | pVal = pH->aRhs[iCons]; |
3924 | } |
3925 | *ppVal = pVal; |
3926 | |
3927 | if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */ |
3928 | rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */ |
3929 | } |
3930 | |
3931 | return rc; |
3932 | } |
3933 | |
3934 | /* |
3935 | ** Return true if ORDER BY clause may be handled as DISTINCT. |
3936 | */ |
3937 | int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ |
3938 | HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; |
3939 | assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 ); |
3940 | return pHidden->eDistinct; |
3941 | } |
3942 | |
3943 | #if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ |
3944 | && !defined(SQLITE_OMIT_VIRTUALTABLE) |
3945 | /* |
3946 | ** Cause the prepared statement that is associated with a call to |
3947 | ** xBestIndex to potentiall use all schemas. If the statement being |
3948 | ** prepared is read-only, then just start read transactions on all |
3949 | ** schemas. But if this is a write operation, start writes on all |
3950 | ** schemas. |
3951 | ** |
3952 | ** This is used by the (built-in) sqlite_dbpage virtual table. |
3953 | */ |
3954 | void sqlite3VtabUsesAllSchemas(sqlite3_index_info *pIdxInfo){ |
3955 | HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; |
3956 | Parse *pParse = pHidden->pParse; |
3957 | int nDb = pParse->db->nDb; |
3958 | int i; |
3959 | for(i=0; i<nDb; i++){ |
3960 | sqlite3CodeVerifySchema(pParse, i); |
3961 | } |
3962 | if( pParse->writeMask ){ |
3963 | for(i=0; i<nDb; i++){ |
3964 | sqlite3BeginWriteOperation(pParse, 0, i); |
3965 | } |
3966 | } |
3967 | } |
3968 | #endif |
3969 | |
3970 | /* |
3971 | ** Add all WhereLoop objects for a table of the join identified by |
3972 | ** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. |
3973 | ** |
3974 | ** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and |
3975 | ** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause |
3976 | ** entries that occur before the virtual table in the FROM clause and are |
3977 | ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the |
3978 | ** mUnusable mask contains all FROM clause entries that occur after the |
3979 | ** virtual table and are separated from it by at least one LEFT or |
3980 | ** CROSS JOIN. |
3981 | ** |
3982 | ** For example, if the query were: |
3983 | ** |
3984 | ** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6; |
3985 | ** |
3986 | ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6). |
3987 | ** |
3988 | ** All the tables in mPrereq must be scanned before the current virtual |
3989 | ** table. So any terms for which all prerequisites are satisfied by |
3990 | ** mPrereq may be specified as "usable" in all calls to xBestIndex. |
3991 | ** Conversely, all tables in mUnusable must be scanned after the current |
3992 | ** virtual table, so any terms for which the prerequisites overlap with |
3993 | ** mUnusable should always be configured as "not-usable" for xBestIndex. |
3994 | */ |
3995 | static int whereLoopAddVirtual( |
3996 | WhereLoopBuilder *pBuilder, /* WHERE clause information */ |
3997 | Bitmask mPrereq, /* Tables that must be scanned before this one */ |
3998 | Bitmask mUnusable /* Tables that must be scanned after this one */ |
3999 | ){ |
4000 | int rc = SQLITE_OK; /* Return code */ |
4001 | WhereInfo *pWInfo; /* WHERE analysis context */ |
4002 | Parse *pParse; /* The parsing context */ |
4003 | WhereClause *pWC; /* The WHERE clause */ |
4004 | SrcItem *pSrc; /* The FROM clause term to search */ |
4005 | sqlite3_index_info *p; /* Object to pass to xBestIndex() */ |
4006 | int nConstraint; /* Number of constraints in p */ |
4007 | int bIn; /* True if plan uses IN(...) operator */ |
4008 | WhereLoop *pNew; |
4009 | Bitmask mBest; /* Tables used by best possible plan */ |
4010 | u16 mNoOmit; |
4011 | int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */ |
4012 | |
4013 | assert( (mPrereq & mUnusable)==0 ); |
4014 | pWInfo = pBuilder->pWInfo; |
4015 | pParse = pWInfo->pParse; |
4016 | pWC = pBuilder->pWC; |
4017 | pNew = pBuilder->pNew; |
4018 | pSrc = &pWInfo->pTabList->a[pNew->iTab]; |
4019 | assert( IsVirtual(pSrc->pTab) ); |
4020 | p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit); |
4021 | if( p==0 ) return SQLITE_NOMEM_BKPT; |
4022 | pNew->rSetup = 0; |
4023 | pNew->wsFlags = WHERE_VIRTUALTABLE; |
4024 | pNew->nLTerm = 0; |
4025 | pNew->u.vtab.needFree = 0; |
4026 | nConstraint = p->nConstraint; |
4027 | if( whereLoopResize(pParse->db, pNew, nConstraint) ){ |
4028 | freeIndexInfo(pParse->db, p); |
4029 | return SQLITE_NOMEM_BKPT; |
4030 | } |
4031 | |
4032 | /* First call xBestIndex() with all constraints usable. */ |
4033 | WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n" , pSrc->pTab->zName)); |
4034 | WHERETRACE(0x40, (" VirtualOne: all usable\n" )); |
4035 | rc = whereLoopAddVirtualOne( |
4036 | pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry |
4037 | ); |
4038 | if( bRetry ){ |
4039 | assert( rc==SQLITE_OK ); |
4040 | rc = whereLoopAddVirtualOne( |
4041 | pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0 |
4042 | ); |
4043 | } |
4044 | |
4045 | /* If the call to xBestIndex() with all terms enabled produced a plan |
4046 | ** that does not require any source tables (IOW: a plan with mBest==0) |
4047 | ** and does not use an IN(...) operator, then there is no point in making |
4048 | ** any further calls to xBestIndex() since they will all return the same |
4049 | ** result (if the xBestIndex() implementation is sane). */ |
4050 | if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){ |
4051 | int seenZero = 0; /* True if a plan with no prereqs seen */ |
4052 | int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ |
4053 | Bitmask mPrev = 0; |
4054 | Bitmask mBestNoIn = 0; |
4055 | |
4056 | /* If the plan produced by the earlier call uses an IN(...) term, call |
4057 | ** xBestIndex again, this time with IN(...) terms disabled. */ |
4058 | if( bIn ){ |
4059 | WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n" )); |
4060 | rc = whereLoopAddVirtualOne( |
4061 | pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0); |
4062 | assert( bIn==0 ); |
4063 | mBestNoIn = pNew->prereq & ~mPrereq; |
4064 | if( mBestNoIn==0 ){ |
4065 | seenZero = 1; |
4066 | seenZeroNoIN = 1; |
4067 | } |
4068 | } |
4069 | |
4070 | /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) |
4071 | ** in the set of terms that apply to the current virtual table. */ |
4072 | while( rc==SQLITE_OK ){ |
4073 | int i; |
4074 | Bitmask mNext = ALLBITS; |
4075 | assert( mNext>0 ); |
4076 | for(i=0; i<nConstraint; i++){ |
4077 | Bitmask mThis = ( |
4078 | pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq |
4079 | ); |
4080 | if( mThis>mPrev && mThis<mNext ) mNext = mThis; |
4081 | } |
4082 | mPrev = mNext; |
4083 | if( mNext==ALLBITS ) break; |
4084 | if( mNext==mBest || mNext==mBestNoIn ) continue; |
4085 | WHERETRACE(0x40, (" VirtualOne: mPrev=%04llx mNext=%04llx\n" , |
4086 | (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext)); |
4087 | rc = whereLoopAddVirtualOne( |
4088 | pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0); |
4089 | if( pNew->prereq==mPrereq ){ |
4090 | seenZero = 1; |
4091 | if( bIn==0 ) seenZeroNoIN = 1; |
4092 | } |
4093 | } |
4094 | |
4095 | /* If the calls to xBestIndex() in the above loop did not find a plan |
4096 | ** that requires no source tables at all (i.e. one guaranteed to be |
4097 | ** usable), make a call here with all source tables disabled */ |
4098 | if( rc==SQLITE_OK && seenZero==0 ){ |
4099 | WHERETRACE(0x40, (" VirtualOne: all disabled\n" )); |
4100 | rc = whereLoopAddVirtualOne( |
4101 | pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0); |
4102 | if( bIn==0 ) seenZeroNoIN = 1; |
4103 | } |
4104 | |
4105 | /* If the calls to xBestIndex() have so far failed to find a plan |
4106 | ** that requires no source tables at all and does not use an IN(...) |
4107 | ** operator, make a final call to obtain one here. */ |
4108 | if( rc==SQLITE_OK && seenZeroNoIN==0 ){ |
4109 | WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n" )); |
4110 | rc = whereLoopAddVirtualOne( |
4111 | pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0); |
4112 | } |
4113 | } |
4114 | |
4115 | if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); |
4116 | freeIndexInfo(pParse->db, p); |
4117 | WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n" , pSrc->pTab->zName, rc)); |
4118 | return rc; |
4119 | } |
4120 | #endif /* SQLITE_OMIT_VIRTUALTABLE */ |
4121 | |
4122 | /* |
4123 | ** Add WhereLoop entries to handle OR terms. This works for either |
4124 | ** btrees or virtual tables. |
4125 | */ |
4126 | static int whereLoopAddOr( |
4127 | WhereLoopBuilder *pBuilder, |
4128 | Bitmask mPrereq, |
4129 | Bitmask mUnusable |
4130 | ){ |
4131 | WhereInfo *pWInfo = pBuilder->pWInfo; |
4132 | WhereClause *pWC; |
4133 | WhereLoop *pNew; |
4134 | WhereTerm *pTerm, *pWCEnd; |
4135 | int rc = SQLITE_OK; |
4136 | int iCur; |
4137 | WhereClause tempWC; |
4138 | WhereLoopBuilder sSubBuild; |
4139 | WhereOrSet sSum, sCur; |
4140 | SrcItem *pItem; |
4141 | |
4142 | pWC = pBuilder->pWC; |
4143 | pWCEnd = pWC->a + pWC->nTerm; |
4144 | pNew = pBuilder->pNew; |
4145 | memset(&sSum, 0, sizeof(sSum)); |
4146 | pItem = pWInfo->pTabList->a + pNew->iTab; |
4147 | iCur = pItem->iCursor; |
4148 | |
4149 | /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */ |
4150 | if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK; |
4151 | |
4152 | for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){ |
4153 | if( (pTerm->eOperator & WO_OR)!=0 |
4154 | && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 |
4155 | ){ |
4156 | WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; |
4157 | WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; |
4158 | WhereTerm *pOrTerm; |
4159 | int once = 1; |
4160 | int i, j; |
4161 | |
4162 | sSubBuild = *pBuilder; |
4163 | sSubBuild.pOrSet = &sCur; |
4164 | |
4165 | WHERETRACE(0x200, ("Begin processing OR-clause %p\n" , pTerm)); |
4166 | for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){ |
4167 | if( (pOrTerm->eOperator & WO_AND)!=0 ){ |
4168 | sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; |
4169 | }else if( pOrTerm->leftCursor==iCur ){ |
4170 | tempWC.pWInfo = pWC->pWInfo; |
4171 | tempWC.pOuter = pWC; |
4172 | tempWC.op = TK_AND; |
4173 | tempWC.nTerm = 1; |
4174 | tempWC.nBase = 1; |
4175 | tempWC.a = pOrTerm; |
4176 | sSubBuild.pWC = &tempWC; |
4177 | }else{ |
4178 | continue; |
4179 | } |
4180 | sCur.n = 0; |
4181 | #ifdef WHERETRACE_ENABLED |
4182 | WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n" , |
4183 | (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); |
4184 | if( sqlite3WhereTrace & 0x400 ){ |
4185 | sqlite3WhereClausePrint(sSubBuild.pWC); |
4186 | } |
4187 | #endif |
4188 | #ifndef SQLITE_OMIT_VIRTUALTABLE |
4189 | if( IsVirtual(pItem->pTab) ){ |
4190 | rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); |
4191 | }else |
4192 | #endif |
4193 | { |
4194 | rc = whereLoopAddBtree(&sSubBuild, mPrereq); |
4195 | } |
4196 | if( rc==SQLITE_OK ){ |
4197 | rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); |
4198 | } |
4199 | assert( rc==SQLITE_OK || rc==SQLITE_DONE || sCur.n==0 |
4200 | || rc==SQLITE_NOMEM ); |
4201 | testcase( rc==SQLITE_NOMEM && sCur.n>0 ); |
4202 | testcase( rc==SQLITE_DONE ); |
4203 | if( sCur.n==0 ){ |
4204 | sSum.n = 0; |
4205 | break; |
4206 | }else if( once ){ |
4207 | whereOrMove(&sSum, &sCur); |
4208 | once = 0; |
4209 | }else{ |
4210 | WhereOrSet sPrev; |
4211 | whereOrMove(&sPrev, &sSum); |
4212 | sSum.n = 0; |
4213 | for(i=0; i<sPrev.n; i++){ |
4214 | for(j=0; j<sCur.n; j++){ |
4215 | whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq, |
4216 | sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun), |
4217 | sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut)); |
4218 | } |
4219 | } |
4220 | } |
4221 | } |
4222 | pNew->nLTerm = 1; |
4223 | pNew->aLTerm[0] = pTerm; |
4224 | pNew->wsFlags = WHERE_MULTI_OR; |
4225 | pNew->rSetup = 0; |
4226 | pNew->iSortIdx = 0; |
4227 | memset(&pNew->u, 0, sizeof(pNew->u)); |
4228 | for(i=0; rc==SQLITE_OK && i<sSum.n; i++){ |
4229 | /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs |
4230 | ** of all sub-scans required by the OR-scan. However, due to rounding |
4231 | ** errors, it may be that the cost of the OR-scan is equal to its |
4232 | ** most expensive sub-scan. Add the smallest possible penalty |
4233 | ** (equivalent to multiplying the cost by 1.07) to ensure that |
4234 | ** this does not happen. Otherwise, for WHERE clauses such as the |
4235 | ** following where there is an index on "y": |
4236 | ** |
4237 | ** WHERE likelihood(x=?, 0.99) OR y=? |
4238 | ** |
4239 | ** the planner may elect to "OR" together a full-table scan and an |
4240 | ** index lookup. And other similarly odd results. */ |
4241 | pNew->rRun = sSum.a[i].rRun + 1; |
4242 | pNew->nOut = sSum.a[i].nOut; |
4243 | pNew->prereq = sSum.a[i].prereq; |
4244 | rc = whereLoopInsert(pBuilder, pNew); |
4245 | } |
4246 | WHERETRACE(0x200, ("End processing OR-clause %p\n" , pTerm)); |
4247 | } |
4248 | } |
4249 | return rc; |
4250 | } |
4251 | |
4252 | /* |
4253 | ** Add all WhereLoop objects for all tables |
4254 | */ |
4255 | static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ |
4256 | WhereInfo *pWInfo = pBuilder->pWInfo; |
4257 | Bitmask mPrereq = 0; |
4258 | Bitmask mPrior = 0; |
4259 | int iTab; |
4260 | SrcList *pTabList = pWInfo->pTabList; |
4261 | SrcItem *pItem; |
4262 | SrcItem *pEnd = &pTabList->a[pWInfo->nLevel]; |
4263 | sqlite3 *db = pWInfo->pParse->db; |
4264 | int rc = SQLITE_OK; |
4265 | int bFirstPastRJ = 0; |
4266 | int hasRightJoin = 0; |
4267 | WhereLoop *pNew; |
4268 | |
4269 | |
4270 | /* Loop over the tables in the join, from left to right */ |
4271 | pNew = pBuilder->pNew; |
4272 | |
4273 | /* Verify that pNew has already been initialized */ |
4274 | assert( pNew->nLTerm==0 ); |
4275 | assert( pNew->wsFlags==0 ); |
4276 | assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) ); |
4277 | assert( pNew->aLTerm!=0 ); |
4278 | |
4279 | pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; |
4280 | for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){ |
4281 | Bitmask mUnusable = 0; |
4282 | pNew->iTab = iTab; |
4283 | pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR; |
4284 | pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); |
4285 | if( bFirstPastRJ |
4286 | || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0 |
4287 | ){ |
4288 | /* Add prerequisites to prevent reordering of FROM clause terms |
4289 | ** across CROSS joins and outer joins. The bFirstPastRJ boolean |
4290 | ** prevents the right operand of a RIGHT JOIN from being swapped with |
4291 | ** other elements even further to the right. |
4292 | ** |
4293 | ** The JT_LTORJ case and the hasRightJoin flag work together to |
4294 | ** prevent FROM-clause terms from moving from the right side of |
4295 | ** a LEFT JOIN over to the left side of that join if the LEFT JOIN |
4296 | ** is itself on the left side of a RIGHT JOIN. |
4297 | */ |
4298 | if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; |
4299 | mPrereq |= mPrior; |
4300 | bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; |
4301 | }else if( !hasRightJoin ){ |
4302 | mPrereq = 0; |
4303 | } |
4304 | #ifndef SQLITE_OMIT_VIRTUALTABLE |
4305 | if( IsVirtual(pItem->pTab) ){ |
4306 | SrcItem *p; |
4307 | for(p=&pItem[1]; p<pEnd; p++){ |
4308 | if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){ |
4309 | mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); |
4310 | } |
4311 | } |
4312 | rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable); |
4313 | }else |
4314 | #endif /* SQLITE_OMIT_VIRTUALTABLE */ |
4315 | { |
4316 | rc = whereLoopAddBtree(pBuilder, mPrereq); |
4317 | } |
4318 | if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){ |
4319 | rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); |
4320 | } |
4321 | mPrior |= pNew->maskSelf; |
4322 | if( rc || db->mallocFailed ){ |
4323 | if( rc==SQLITE_DONE ){ |
4324 | /* We hit the query planner search limit set by iPlanLimit */ |
4325 | sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search" ); |
4326 | rc = SQLITE_OK; |
4327 | }else{ |
4328 | break; |
4329 | } |
4330 | } |
4331 | } |
4332 | |
4333 | whereLoopClear(db, pNew); |
4334 | return rc; |
4335 | } |
4336 | |
4337 | /* |
4338 | ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th |
4339 | ** parameters) to see if it outputs rows in the requested ORDER BY |
4340 | ** (or GROUP BY) without requiring a separate sort operation. Return N: |
4341 | ** |
4342 | ** N>0: N terms of the ORDER BY clause are satisfied |
4343 | ** N==0: No terms of the ORDER BY clause are satisfied |
4344 | ** N<0: Unknown yet how many terms of ORDER BY might be satisfied. |
4345 | ** |
4346 | ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as |
4347 | ** strict. With GROUP BY and DISTINCT the only requirement is that |
4348 | ** equivalent rows appear immediately adjacent to one another. GROUP BY |
4349 | ** and DISTINCT do not require rows to appear in any particular order as long |
4350 | ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT |
4351 | ** the pOrderBy terms can be matched in any order. With ORDER BY, the |
4352 | ** pOrderBy terms must be matched in strict left-to-right order. |
4353 | */ |
4354 | static i8 wherePathSatisfiesOrderBy( |
4355 | WhereInfo *pWInfo, /* The WHERE clause */ |
4356 | ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ |
4357 | WherePath *pPath, /* The WherePath to check */ |
4358 | u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */ |
4359 | u16 nLoop, /* Number of entries in pPath->aLoop[] */ |
4360 | WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ |
4361 | Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ |
4362 | ){ |
4363 | u8 revSet; /* True if rev is known */ |
4364 | u8 rev; /* Composite sort order */ |
4365 | u8 revIdx; /* Index sort order */ |
4366 | u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ |
4367 | u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ |
4368 | u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ |
4369 | u16 eqOpMask; /* Allowed equality operators */ |
4370 | u16 nKeyCol; /* Number of key columns in pIndex */ |
4371 | u16 nColumn; /* Total number of ordered columns in the index */ |
4372 | u16 nOrderBy; /* Number terms in the ORDER BY clause */ |
4373 | int iLoop; /* Index of WhereLoop in pPath being processed */ |
4374 | int i, j; /* Loop counters */ |
4375 | int iCur; /* Cursor number for current WhereLoop */ |
4376 | int iColumn; /* A column number within table iCur */ |
4377 | WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ |
4378 | WhereTerm *pTerm; /* A single term of the WHERE clause */ |
4379 | Expr *pOBExpr; /* An expression from the ORDER BY clause */ |
4380 | CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ |
4381 | Index *pIndex; /* The index associated with pLoop */ |
4382 | sqlite3 *db = pWInfo->pParse->db; /* Database connection */ |
4383 | Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ |
4384 | Bitmask obDone; /* Mask of all ORDER BY terms */ |
4385 | Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ |
4386 | Bitmask ready; /* Mask of inner loops */ |
4387 | |
4388 | /* |
4389 | ** We say the WhereLoop is "one-row" if it generates no more than one |
4390 | ** row of output. A WhereLoop is one-row if all of the following are true: |
4391 | ** (a) All index columns match with WHERE_COLUMN_EQ. |
4392 | ** (b) The index is unique |
4393 | ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. |
4394 | ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. |
4395 | ** |
4396 | ** We say the WhereLoop is "order-distinct" if the set of columns from |
4397 | ** that WhereLoop that are in the ORDER BY clause are different for every |
4398 | ** row of the WhereLoop. Every one-row WhereLoop is automatically |
4399 | ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause |
4400 | ** is not order-distinct. To be order-distinct is not quite the same as being |
4401 | ** UNIQUE since a UNIQUE column or index can have multiple rows that |
4402 | ** are NULL and NULL values are equivalent for the purpose of order-distinct. |
4403 | ** To be order-distinct, the columns must be UNIQUE and NOT NULL. |
4404 | ** |
4405 | ** The rowid for a table is always UNIQUE and NOT NULL so whenever the |
4406 | ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is |
4407 | ** automatically order-distinct. |
4408 | */ |
4409 | |
4410 | assert( pOrderBy!=0 ); |
4411 | if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; |
4412 | |
4413 | nOrderBy = pOrderBy->nExpr; |
4414 | testcase( nOrderBy==BMS-1 ); |
4415 | if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ |
4416 | isOrderDistinct = 1; |
4417 | obDone = MASKBIT(nOrderBy)-1; |
4418 | orderDistinctMask = 0; |
4419 | ready = 0; |
4420 | eqOpMask = WO_EQ | WO_IS | WO_ISNULL; |
4421 | if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){ |
4422 | eqOpMask |= WO_IN; |
4423 | } |
4424 | for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){ |
4425 | if( iLoop>0 ) ready |= pLoop->maskSelf; |
4426 | if( iLoop<nLoop ){ |
4427 | pLoop = pPath->aLoop[iLoop]; |
4428 | if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue; |
4429 | }else{ |
4430 | pLoop = pLast; |
4431 | } |
4432 | if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ |
4433 | if( pLoop->u.vtab.isOrdered |
4434 | && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) |
4435 | ){ |
4436 | obSat = obDone; |
4437 | } |
4438 | break; |
4439 | }else if( wctrlFlags & WHERE_DISTINCTBY ){ |
4440 | pLoop->u.btree.nDistinctCol = 0; |
4441 | } |
4442 | iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; |
4443 | |
4444 | /* Mark off any ORDER BY term X that is a column in the table of |
4445 | ** the current loop for which there is term in the WHERE |
4446 | ** clause of the form X IS NULL or X=? that reference only outer |
4447 | ** loops. |
4448 | */ |
4449 | for(i=0; i<nOrderBy; i++){ |
4450 | if( MASKBIT(i) & obSat ) continue; |
4451 | pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); |
4452 | if( NEVER(pOBExpr==0) ) continue; |
4453 | if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; |
4454 | if( pOBExpr->iTable!=iCur ) continue; |
4455 | pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, |
4456 | ~ready, eqOpMask, 0); |
4457 | if( pTerm==0 ) continue; |
4458 | if( pTerm->eOperator==WO_IN ){ |
4459 | /* IN terms are only valid for sorting in the ORDER BY LIMIT |
4460 | ** optimization, and then only if they are actually used |
4461 | ** by the query plan */ |
4462 | assert( wctrlFlags & |
4463 | (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ); |
4464 | for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){} |
4465 | if( j>=pLoop->nLTerm ) continue; |
4466 | } |
4467 | if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ |
4468 | Parse *pParse = pWInfo->pParse; |
4469 | CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr); |
4470 | CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); |
4471 | assert( pColl1 ); |
4472 | if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){ |
4473 | continue; |
4474 | } |
4475 | testcase( pTerm->pExpr->op==TK_IS ); |
4476 | } |
4477 | obSat |= MASKBIT(i); |
4478 | } |
4479 | |
4480 | if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ |
4481 | if( pLoop->wsFlags & WHERE_IPK ){ |
4482 | pIndex = 0; |
4483 | nKeyCol = 0; |
4484 | nColumn = 1; |
4485 | }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ |
4486 | return 0; |
4487 | }else{ |
4488 | nKeyCol = pIndex->nKeyCol; |
4489 | nColumn = pIndex->nColumn; |
4490 | assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); |
4491 | assert( pIndex->aiColumn[nColumn-1]==XN_ROWID |
4492 | || !HasRowid(pIndex->pTable)); |
4493 | /* All relevant terms of the index must also be non-NULL in order |
4494 | ** for isOrderDistinct to be true. So the isOrderDistint value |
4495 | ** computed here might be a false positive. Corrections will be |
4496 | ** made at tag-20210426-1 below */ |
4497 | isOrderDistinct = IsUniqueIndex(pIndex) |
4498 | && (pLoop->wsFlags & WHERE_SKIPSCAN)==0; |
4499 | } |
4500 | |
4501 | /* Loop through all columns of the index and deal with the ones |
4502 | ** that are not constrained by == or IN. |
4503 | */ |
4504 | rev = revSet = 0; |
4505 | distinctColumns = 0; |
4506 | for(j=0; j<nColumn; j++){ |
4507 | u8 bOnce = 1; /* True to run the ORDER BY search loop */ |
4508 | |
4509 | assert( j>=pLoop->u.btree.nEq |
4510 | || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip) |
4511 | ); |
4512 | if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){ |
4513 | u16 eOp = pLoop->aLTerm[j]->eOperator; |
4514 | |
4515 | /* Skip over == and IS and ISNULL terms. (Also skip IN terms when |
4516 | ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL |
4517 | ** terms imply that the index is not UNIQUE NOT NULL in which case |
4518 | ** the loop need to be marked as not order-distinct because it can |
4519 | ** have repeated NULL rows. |
4520 | ** |
4521 | ** If the current term is a column of an ((?,?) IN (SELECT...)) |
4522 | ** expression for which the SELECT returns more than one column, |
4523 | ** check that it is the only column used by this loop. Otherwise, |
4524 | ** if it is one of two or more, none of the columns can be |
4525 | ** considered to match an ORDER BY term. |
4526 | */ |
4527 | if( (eOp & eqOpMask)!=0 ){ |
4528 | if( eOp & (WO_ISNULL|WO_IS) ){ |
4529 | testcase( eOp & WO_ISNULL ); |
4530 | testcase( eOp & WO_IS ); |
4531 | testcase( isOrderDistinct ); |
4532 | isOrderDistinct = 0; |
4533 | } |
4534 | continue; |
4535 | }else if( ALWAYS(eOp & WO_IN) ){ |
4536 | /* ALWAYS() justification: eOp is an equality operator due to the |
4537 | ** j<pLoop->u.btree.nEq constraint above. Any equality other |
4538 | ** than WO_IN is captured by the previous "if". So this one |
4539 | ** always has to be WO_IN. */ |
4540 | Expr *pX = pLoop->aLTerm[j]->pExpr; |
4541 | for(i=j+1; i<pLoop->u.btree.nEq; i++){ |
4542 | if( pLoop->aLTerm[i]->pExpr==pX ){ |
4543 | assert( (pLoop->aLTerm[i]->eOperator & WO_IN) ); |
4544 | bOnce = 0; |
4545 | break; |
4546 | } |
4547 | } |
4548 | } |
4549 | } |
4550 | |
4551 | /* Get the column number in the table (iColumn) and sort order |
4552 | ** (revIdx) for the j-th column of the index. |
4553 | */ |
4554 | if( pIndex ){ |
4555 | iColumn = pIndex->aiColumn[j]; |
4556 | revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC; |
4557 | if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID; |
4558 | }else{ |
4559 | iColumn = XN_ROWID; |
4560 | revIdx = 0; |
4561 | } |
4562 | |
4563 | /* An unconstrained column that might be NULL means that this |
4564 | ** WhereLoop is not well-ordered. tag-20210426-1 |
4565 | */ |
4566 | if( isOrderDistinct ){ |
4567 | if( iColumn>=0 |
4568 | && j>=pLoop->u.btree.nEq |
4569 | && pIndex->pTable->aCol[iColumn].notNull==0 |
4570 | ){ |
4571 | isOrderDistinct = 0; |
4572 | } |
4573 | if( iColumn==XN_EXPR ){ |
4574 | isOrderDistinct = 0; |
4575 | } |
4576 | } |
4577 | |
4578 | /* Find the ORDER BY term that corresponds to the j-th column |
4579 | ** of the index and mark that ORDER BY term off |
4580 | */ |
4581 | isMatch = 0; |
4582 | for(i=0; bOnce && i<nOrderBy; i++){ |
4583 | if( MASKBIT(i) & obSat ) continue; |
4584 | pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); |
4585 | testcase( wctrlFlags & WHERE_GROUPBY ); |
4586 | testcase( wctrlFlags & WHERE_DISTINCTBY ); |
4587 | if( NEVER(pOBExpr==0) ) continue; |
4588 | if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; |
4589 | if( iColumn>=XN_ROWID ){ |
4590 | if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; |
4591 | if( pOBExpr->iTable!=iCur ) continue; |
4592 | if( pOBExpr->iColumn!=iColumn ) continue; |
4593 | }else{ |
4594 | Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr; |
4595 | if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){ |
4596 | continue; |
4597 | } |
4598 | } |
4599 | if( iColumn!=XN_ROWID ){ |
4600 | pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); |
4601 | if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; |
4602 | } |
4603 | if( wctrlFlags & WHERE_DISTINCTBY ){ |
4604 | pLoop->u.btree.nDistinctCol = j+1; |
4605 | } |
4606 | isMatch = 1; |
4607 | break; |
4608 | } |
4609 | if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ |
4610 | /* Make sure the sort order is compatible in an ORDER BY clause. |
4611 | ** Sort order is irrelevant for a GROUP BY clause. */ |
4612 | if( revSet ){ |
4613 | if( (rev ^ revIdx) |
4614 | != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC) |
4615 | ){ |
4616 | isMatch = 0; |
4617 | } |
4618 | }else{ |
4619 | rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC); |
4620 | if( rev ) *pRevMask |= MASKBIT(iLoop); |
4621 | revSet = 1; |
4622 | } |
4623 | } |
4624 | if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){ |
4625 | if( j==pLoop->u.btree.nEq ){ |
4626 | pLoop->wsFlags |= WHERE_BIGNULL_SORT; |
4627 | }else{ |
4628 | isMatch = 0; |
4629 | } |
4630 | } |
4631 | if( isMatch ){ |
4632 | if( iColumn==XN_ROWID ){ |
4633 | testcase( distinctColumns==0 ); |
4634 | distinctColumns = 1; |
4635 | } |
4636 | obSat |= MASKBIT(i); |
4637 | }else{ |
4638 | /* No match found */ |
4639 | if( j==0 || j<nKeyCol ){ |
4640 | testcase( isOrderDistinct!=0 ); |
4641 | isOrderDistinct = 0; |
4642 | } |
4643 | break; |
4644 | } |
4645 | } /* end Loop over all index columns */ |
4646 | if( distinctColumns ){ |
4647 | testcase( isOrderDistinct==0 ); |
4648 | isOrderDistinct = 1; |
4649 | } |
4650 | } /* end-if not one-row */ |
4651 | |
4652 | /* Mark off any other ORDER BY terms that reference pLoop */ |
4653 | if( isOrderDistinct ){ |
4654 | orderDistinctMask |= pLoop->maskSelf; |
4655 | for(i=0; i<nOrderBy; i++){ |
4656 | Expr *p; |
4657 | Bitmask mTerm; |
4658 | if( MASKBIT(i) & obSat ) continue; |
4659 | p = pOrderBy->a[i].pExpr; |
4660 | mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); |
4661 | if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; |
4662 | if( (mTerm&~orderDistinctMask)==0 ){ |
4663 | obSat |= MASKBIT(i); |
4664 | } |
4665 | } |
4666 | } |
4667 | } /* End the loop over all WhereLoops from outer-most down to inner-most */ |
4668 | if( obSat==obDone ) return (i8)nOrderBy; |
4669 | if( !isOrderDistinct ){ |
4670 | for(i=nOrderBy-1; i>0; i--){ |
4671 | Bitmask m = ALWAYS(i<BMS) ? MASKBIT(i) - 1 : 0; |
4672 | if( (obSat&m)==m ) return i; |
4673 | } |
4674 | return 0; |
4675 | } |
4676 | return -1; |
4677 | } |
4678 | |
4679 | |
4680 | /* |
4681 | ** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), |
4682 | ** the planner assumes that the specified pOrderBy list is actually a GROUP |
4683 | ** BY clause - and so any order that groups rows as required satisfies the |
4684 | ** request. |
4685 | ** |
4686 | ** Normally, in this case it is not possible for the caller to determine |
4687 | ** whether or not the rows are really being delivered in sorted order, or |
4688 | ** just in some other order that provides the required grouping. However, |
4689 | ** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then |
4690 | ** this function may be called on the returned WhereInfo object. It returns |
4691 | ** true if the rows really will be sorted in the specified order, or false |
4692 | ** otherwise. |
4693 | ** |
4694 | ** For example, assuming: |
4695 | ** |
4696 | ** CREATE INDEX i1 ON t1(x, Y); |
4697 | ** |
4698 | ** then |
4699 | ** |
4700 | ** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 |
4701 | ** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 |
4702 | */ |
4703 | int sqlite3WhereIsSorted(WhereInfo *pWInfo){ |
4704 | assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) ); |
4705 | assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); |
4706 | return pWInfo->sorted; |
4707 | } |
4708 | |
4709 | #ifdef WHERETRACE_ENABLED |
4710 | /* For debugging use only: */ |
4711 | static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ |
4712 | static char zName[65]; |
4713 | int i; |
4714 | for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; } |
4715 | if( pLast ) zName[i++] = pLast->cId; |
4716 | zName[i] = 0; |
4717 | return zName; |
4718 | } |
4719 | #endif |
4720 | |
4721 | /* |
4722 | ** Return the cost of sorting nRow rows, assuming that the keys have |
4723 | ** nOrderby columns and that the first nSorted columns are already in |
4724 | ** order. |
4725 | */ |
4726 | static LogEst whereSortingCost( |
4727 | WhereInfo *pWInfo, |
4728 | LogEst nRow, |
4729 | int nOrderBy, |
4730 | int nSorted |
4731 | ){ |
4732 | /* TUNING: Estimated cost of a full external sort, where N is |
4733 | ** the number of rows to sort is: |
4734 | ** |
4735 | ** cost = (3.0 * N * log(N)). |
4736 | ** |
4737 | ** Or, if the order-by clause has X terms but only the last Y |
4738 | ** terms are out of order, then block-sorting will reduce the |
4739 | ** sorting cost to: |
4740 | ** |
4741 | ** cost = (3.0 * N * log(N)) * (Y/X) |
4742 | ** |
4743 | ** The (Y/X) term is implemented using stack variable rScale |
4744 | ** below. |
4745 | */ |
4746 | LogEst rScale, rSortCost; |
4747 | assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); |
4748 | rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; |
4749 | rSortCost = nRow + rScale + 16; |
4750 | |
4751 | /* Multiple by log(M) where M is the number of output rows. |
4752 | ** Use the LIMIT for M if it is smaller. Or if this sort is for |
4753 | ** a DISTINCT operator, M will be the number of distinct output |
4754 | ** rows, so fudge it downwards a bit. |
4755 | */ |
4756 | if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimit<nRow ){ |
4757 | nRow = pWInfo->iLimit; |
4758 | }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){ |
4759 | /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT |
4760 | ** reduces the number of output rows by a factor of 2 */ |
4761 | if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); } |
4762 | } |
4763 | rSortCost += estLog(nRow); |
4764 | return rSortCost; |
4765 | } |
4766 | |
4767 | /* |
4768 | ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine |
4769 | ** attempts to find the lowest cost path that visits each WhereLoop |
4770 | ** once. This path is then loaded into the pWInfo->a[].pWLoop fields. |
4771 | ** |
4772 | ** Assume that the total number of output rows that will need to be sorted |
4773 | ** will be nRowEst (in the 10*log2 representation). Or, ignore sorting |
4774 | ** costs if nRowEst==0. |
4775 | ** |
4776 | ** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation |
4777 | ** error occurs. |
4778 | */ |
4779 | static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ |
4780 | int mxChoice; /* Maximum number of simultaneous paths tracked */ |
4781 | int nLoop; /* Number of terms in the join */ |
4782 | Parse *pParse; /* Parsing context */ |
4783 | int iLoop; /* Loop counter over the terms of the join */ |
4784 | int ii, jj; /* Loop counters */ |
4785 | int mxI = 0; /* Index of next entry to replace */ |
4786 | int nOrderBy; /* Number of ORDER BY clause terms */ |
4787 | LogEst mxCost = 0; /* Maximum cost of a set of paths */ |
4788 | LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ |
4789 | int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ |
4790 | WherePath *aFrom; /* All nFrom paths at the previous level */ |
4791 | WherePath *aTo; /* The nTo best paths at the current level */ |
4792 | WherePath *pFrom; /* An element of aFrom[] that we are working on */ |
4793 | WherePath *pTo; /* An element of aTo[] that we are working on */ |
4794 | WhereLoop *pWLoop; /* One of the WhereLoop objects */ |
4795 | WhereLoop **pX; /* Used to divy up the pSpace memory */ |
4796 | LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ |
4797 | char *pSpace; /* Temporary memory used by this routine */ |
4798 | int nSpace; /* Bytes of space allocated at pSpace */ |
4799 | |
4800 | pParse = pWInfo->pParse; |
4801 | nLoop = pWInfo->nLevel; |
4802 | /* TUNING: For simple queries, only the best path is tracked. |
4803 | ** For 2-way joins, the 5 best paths are followed. |
4804 | ** For joins of 3 or more tables, track the 10 best paths */ |
4805 | mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); |
4806 | assert( nLoop<=pWInfo->pTabList->nSrc ); |
4807 | WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n" , nRowEst)); |
4808 | |
4809 | /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this |
4810 | ** case the purpose of this call is to estimate the number of rows returned |
4811 | ** by the overall query. Once this estimate has been obtained, the caller |
4812 | ** will invoke this function a second time, passing the estimate as the |
4813 | ** nRowEst parameter. */ |
4814 | if( pWInfo->pOrderBy==0 || nRowEst==0 ){ |
4815 | nOrderBy = 0; |
4816 | }else{ |
4817 | nOrderBy = pWInfo->pOrderBy->nExpr; |
4818 | } |
4819 | |
4820 | /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ |
4821 | nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; |
4822 | nSpace += sizeof(LogEst) * nOrderBy; |
4823 | pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace); |
4824 | if( pSpace==0 ) return SQLITE_NOMEM_BKPT; |
4825 | aTo = (WherePath*)pSpace; |
4826 | aFrom = aTo+mxChoice; |
4827 | memset(aFrom, 0, sizeof(aFrom[0])); |
4828 | pX = (WhereLoop**)(aFrom+mxChoice); |
4829 | for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ |
4830 | pFrom->aLoop = pX; |
4831 | } |
4832 | if( nOrderBy ){ |
4833 | /* If there is an ORDER BY clause and it is not being ignored, set up |
4834 | ** space for the aSortCost[] array. Each element of the aSortCost array |
4835 | ** is either zero - meaning it has not yet been initialized - or the |
4836 | ** cost of sorting nRowEst rows of data where the first X terms of |
4837 | ** the ORDER BY clause are already in order, where X is the array |
4838 | ** index. */ |
4839 | aSortCost = (LogEst*)pX; |
4840 | memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); |
4841 | } |
4842 | assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); |
4843 | assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); |
4844 | |
4845 | /* Seed the search with a single WherePath containing zero WhereLoops. |
4846 | ** |
4847 | ** TUNING: Do not let the number of iterations go above 28. If the cost |
4848 | ** of computing an automatic index is not paid back within the first 28 |
4849 | ** rows, then do not use the automatic index. */ |
4850 | aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) ); |
4851 | nFrom = 1; |
4852 | assert( aFrom[0].isOrdered==0 ); |
4853 | if( nOrderBy ){ |
4854 | /* If nLoop is zero, then there are no FROM terms in the query. Since |
4855 | ** in this case the query may return a maximum of one row, the results |
4856 | ** are already in the requested order. Set isOrdered to nOrderBy to |
4857 | ** indicate this. Or, if nLoop is greater than zero, set isOrdered to |
4858 | ** -1, indicating that the result set may or may not be ordered, |
4859 | ** depending on the loops added to the current plan. */ |
4860 | aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; |
4861 | } |
4862 | |
4863 | /* Compute successively longer WherePaths using the previous generation |
4864 | ** of WherePaths as the basis for the next. Keep track of the mxChoice |
4865 | ** best paths at each generation */ |
4866 | for(iLoop=0; iLoop<nLoop; iLoop++){ |
4867 | nTo = 0; |
4868 | for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){ |
4869 | for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ |
4870 | LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ |
4871 | LogEst rCost; /* Cost of path (pFrom+pWLoop) */ |
4872 | LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ |
4873 | i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */ |
4874 | Bitmask maskNew; /* Mask of src visited by (..) */ |
4875 | Bitmask revMask; /* Mask of rev-order loops for (..) */ |
4876 | |
4877 | if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; |
4878 | if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; |
4879 | if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){ |
4880 | /* Do not use an automatic index if the this loop is expected |
4881 | ** to run less than 1.25 times. It is tempting to also exclude |
4882 | ** automatic index usage on an outer loop, but sometimes an automatic |
4883 | ** index is useful in the outer loop of a correlated subquery. */ |
4884 | assert( 10==sqlite3LogEst(2) ); |
4885 | continue; |
4886 | } |
4887 | |
4888 | /* At this point, pWLoop is a candidate to be the next loop. |
4889 | ** Compute its cost */ |
4890 | rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); |
4891 | rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); |
4892 | nOut = pFrom->nRow + pWLoop->nOut; |
4893 | maskNew = pFrom->maskLoop | pWLoop->maskSelf; |
4894 | isOrdered = pFrom->isOrdered; |
4895 | if( isOrdered<0 ){ |
4896 | revMask = 0; |
4897 | isOrdered = wherePathSatisfiesOrderBy(pWInfo, |
4898 | pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, |
4899 | iLoop, pWLoop, &revMask); |
4900 | }else{ |
4901 | revMask = pFrom->revLoop; |
4902 | } |
4903 | if( isOrdered>=0 && isOrdered<nOrderBy ){ |
4904 | if( aSortCost[isOrdered]==0 ){ |
4905 | aSortCost[isOrdered] = whereSortingCost( |
4906 | pWInfo, nRowEst, nOrderBy, isOrdered |
4907 | ); |
4908 | } |
4909 | /* TUNING: Add a small extra penalty (5) to sorting as an |
4910 | ** extra encouragment to the query planner to select a plan |
4911 | ** where the rows emerge in the correct order without any sorting |
4912 | ** required. */ |
4913 | rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 5; |
4914 | |
4915 | WHERETRACE(0x002, |
4916 | ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n" , |
4917 | aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy, |
4918 | rUnsorted, rCost)); |
4919 | }else{ |
4920 | rCost = rUnsorted; |
4921 | rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ |
4922 | } |
4923 | |
4924 | /* TUNING: A full-scan of a VIEW or subquery in the outer loop |
4925 | ** is not so bad. */ |
4926 | if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){ |
4927 | rCost += -10; |
4928 | nOut += -30; |
4929 | } |
4930 | |
4931 | /* Check to see if pWLoop should be added to the set of |
4932 | ** mxChoice best-so-far paths. |
4933 | ** |
4934 | ** First look for an existing path among best-so-far paths |
4935 | ** that covers the same set of loops and has the same isOrdered |
4936 | ** setting as the current path candidate. |
4937 | ** |
4938 | ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent |
4939 | ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range |
4940 | ** of legal values for isOrdered, -1..64. |
4941 | */ |
4942 | for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){ |
4943 | if( pTo->maskLoop==maskNew |
4944 | && ((pTo->isOrdered^isOrdered)&0x80)==0 |
4945 | ){ |
4946 | testcase( jj==nTo-1 ); |
4947 | break; |
4948 | } |
4949 | } |
4950 | if( jj>=nTo ){ |
4951 | /* None of the existing best-so-far paths match the candidate. */ |
4952 | if( nTo>=mxChoice |
4953 | && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) |
4954 | ){ |
4955 | /* The current candidate is no better than any of the mxChoice |
4956 | ** paths currently in the best-so-far buffer. So discard |
4957 | ** this candidate as not viable. */ |
4958 | #ifdef WHERETRACE_ENABLED /* 0x4 */ |
4959 | if( sqlite3WhereTrace&0x4 ){ |
4960 | sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n" , |
4961 | wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, |
4962 | isOrdered>=0 ? isOrdered+'0' : '?'); |
4963 | } |
4964 | #endif |
4965 | continue; |
4966 | } |
4967 | /* If we reach this points it means that the new candidate path |
4968 | ** needs to be added to the set of best-so-far paths. */ |
4969 | if( nTo<mxChoice ){ |
4970 | /* Increase the size of the aTo set by one */ |
4971 | jj = nTo++; |
4972 | }else{ |
4973 | /* New path replaces the prior worst to keep count below mxChoice */ |
4974 | jj = mxI; |
4975 | } |
4976 | pTo = &aTo[jj]; |
4977 | #ifdef WHERETRACE_ENABLED /* 0x4 */ |
4978 | if( sqlite3WhereTrace&0x4 ){ |
4979 | sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n" , |
4980 | wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, |
4981 | isOrdered>=0 ? isOrdered+'0' : '?'); |
4982 | } |
4983 | #endif |
4984 | }else{ |
4985 | /* Control reaches here if best-so-far path pTo=aTo[jj] covers the |
4986 | ** same set of loops and has the same isOrdered setting as the |
4987 | ** candidate path. Check to see if the candidate should replace |
4988 | ** pTo or if the candidate should be skipped. |
4989 | ** |
4990 | ** The conditional is an expanded vector comparison equivalent to: |
4991 | ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted) |
4992 | */ |
4993 | if( pTo->rCost<rCost |
4994 | || (pTo->rCost==rCost |
4995 | && (pTo->nRow<nOut |
4996 | || (pTo->nRow==nOut && pTo->rUnsorted<=rUnsorted) |
4997 | ) |
4998 | ) |
4999 | ){ |
5000 | #ifdef WHERETRACE_ENABLED /* 0x4 */ |
5001 | if( sqlite3WhereTrace&0x4 ){ |
5002 | sqlite3DebugPrintf( |
5003 | "Skip %s cost=%-3d,%3d,%3d order=%c" , |
5004 | wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, |
5005 | isOrdered>=0 ? isOrdered+'0' : '?'); |
5006 | sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n" , |
5007 | wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, |
5008 | pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); |
5009 | } |
5010 | #endif |
5011 | /* Discard the candidate path from further consideration */ |
5012 | testcase( pTo->rCost==rCost ); |
5013 | continue; |
5014 | } |
5015 | testcase( pTo->rCost==rCost+1 ); |
5016 | /* Control reaches here if the candidate path is better than the |
5017 | ** pTo path. Replace pTo with the candidate. */ |
5018 | #ifdef WHERETRACE_ENABLED /* 0x4 */ |
5019 | if( sqlite3WhereTrace&0x4 ){ |
5020 | sqlite3DebugPrintf( |
5021 | "Update %s cost=%-3d,%3d,%3d order=%c" , |
5022 | wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, |
5023 | isOrdered>=0 ? isOrdered+'0' : '?'); |
5024 | sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n" , |
5025 | wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, |
5026 | pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); |
5027 | } |
5028 | #endif |
5029 | } |
5030 | /* pWLoop is a winner. Add it to the set of best so far */ |
5031 | pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; |
5032 | pTo->revLoop = revMask; |
5033 | pTo->nRow = nOut; |
5034 | pTo->rCost = rCost; |
5035 | pTo->rUnsorted = rUnsorted; |
5036 | pTo->isOrdered = isOrdered; |
5037 | memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); |
5038 | pTo->aLoop[iLoop] = pWLoop; |
5039 | if( nTo>=mxChoice ){ |
5040 | mxI = 0; |
5041 | mxCost = aTo[0].rCost; |
5042 | mxUnsorted = aTo[0].nRow; |
5043 | for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){ |
5044 | if( pTo->rCost>mxCost |
5045 | || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) |
5046 | ){ |
5047 | mxCost = pTo->rCost; |
5048 | mxUnsorted = pTo->rUnsorted; |
5049 | mxI = jj; |
5050 | } |
5051 | } |
5052 | } |
5053 | } |
5054 | } |
5055 | |
5056 | #ifdef WHERETRACE_ENABLED /* >=2 */ |
5057 | if( sqlite3WhereTrace & 0x02 ){ |
5058 | sqlite3DebugPrintf("---- after round %d ----\n" , iLoop); |
5059 | for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){ |
5060 | sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c" , |
5061 | wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, |
5062 | pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); |
5063 | if( pTo->isOrdered>0 ){ |
5064 | sqlite3DebugPrintf(" rev=0x%llx\n" , pTo->revLoop); |
5065 | }else{ |
5066 | sqlite3DebugPrintf("\n" ); |
5067 | } |
5068 | } |
5069 | } |
5070 | #endif |
5071 | |
5072 | /* Swap the roles of aFrom and aTo for the next generation */ |
5073 | pFrom = aTo; |
5074 | aTo = aFrom; |
5075 | aFrom = pFrom; |
5076 | nFrom = nTo; |
5077 | } |
5078 | |
5079 | if( nFrom==0 ){ |
5080 | sqlite3ErrorMsg(pParse, "no query solution" ); |
5081 | sqlite3StackFreeNN(pParse->db, pSpace); |
5082 | return SQLITE_ERROR; |
5083 | } |
5084 | |
5085 | /* Find the lowest cost path. pFrom will be left pointing to that path */ |
5086 | pFrom = aFrom; |
5087 | for(ii=1; ii<nFrom; ii++){ |
5088 | if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; |
5089 | } |
5090 | assert( pWInfo->nLevel==nLoop ); |
5091 | /* Load the lowest cost path into pWInfo */ |
5092 | for(iLoop=0; iLoop<nLoop; iLoop++){ |
5093 | WhereLevel *pLevel = pWInfo->a + iLoop; |
5094 | pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; |
5095 | pLevel->iFrom = pWLoop->iTab; |
5096 | pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; |
5097 | } |
5098 | if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 |
5099 | && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 |
5100 | && pWInfo->eDistinct==WHERE_DISTINCT_NOOP |
5101 | && nRowEst |
5102 | ){ |
5103 | Bitmask notUsed; |
5104 | int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, |
5105 | WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); |
5106 | if( rc==pWInfo->pResultSet->nExpr ){ |
5107 | pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; |
5108 | } |
5109 | } |
5110 | pWInfo->bOrderedInnerLoop = 0; |
5111 | if( pWInfo->pOrderBy ){ |
5112 | pWInfo->nOBSat = pFrom->isOrdered; |
5113 | if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ |
5114 | if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ |
5115 | pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; |
5116 | } |
5117 | }else{ |
5118 | pWInfo->revMask = pFrom->revLoop; |
5119 | if( pWInfo->nOBSat<=0 ){ |
5120 | pWInfo->nOBSat = 0; |
5121 | if( nLoop>0 ){ |
5122 | u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags; |
5123 | if( (wsFlags & WHERE_ONEROW)==0 |
5124 | && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN) |
5125 | ){ |
5126 | Bitmask m = 0; |
5127 | int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, |
5128 | WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m); |
5129 | testcase( wsFlags & WHERE_IPK ); |
5130 | testcase( wsFlags & WHERE_COLUMN_IN ); |
5131 | if( rc==pWInfo->pOrderBy->nExpr ){ |
5132 | pWInfo->bOrderedInnerLoop = 1; |
5133 | pWInfo->revMask = m; |
5134 | } |
5135 | } |
5136 | } |
5137 | }else if( nLoop |
5138 | && pWInfo->nOBSat==1 |
5139 | && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0 |
5140 | ){ |
5141 | pWInfo->bOrderedInnerLoop = 1; |
5142 | } |
5143 | } |
5144 | if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) |
5145 | && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 |
5146 | ){ |
5147 | Bitmask revMask = 0; |
5148 | int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, |
5149 | pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask |
5150 | ); |
5151 | assert( pWInfo->sorted==0 ); |
5152 | if( nOrder==pWInfo->pOrderBy->nExpr ){ |
5153 | pWInfo->sorted = 1; |
5154 | pWInfo->revMask = revMask; |
5155 | } |
5156 | } |
5157 | } |
5158 | |
5159 | |
5160 | pWInfo->nRowOut = pFrom->nRow; |
5161 | |
5162 | /* Free temporary memory and return success */ |
5163 | sqlite3StackFreeNN(pParse->db, pSpace); |
5164 | return SQLITE_OK; |
5165 | } |
5166 | |
5167 | /* |
5168 | ** Most queries use only a single table (they are not joins) and have |
5169 | ** simple == constraints against indexed fields. This routine attempts |
5170 | ** to plan those simple cases using much less ceremony than the |
5171 | ** general-purpose query planner, and thereby yield faster sqlite3_prepare() |
5172 | ** times for the common case. |
5173 | ** |
5174 | ** Return non-zero on success, if this query can be handled by this |
5175 | ** no-frills query planner. Return zero if this query needs the |
5176 | ** general-purpose query planner. |
5177 | */ |
5178 | static int whereShortCut(WhereLoopBuilder *pBuilder){ |
5179 | WhereInfo *pWInfo; |
5180 | SrcItem *pItem; |
5181 | WhereClause *pWC; |
5182 | WhereTerm *pTerm; |
5183 | WhereLoop *pLoop; |
5184 | int iCur; |
5185 | int j; |
5186 | Table *pTab; |
5187 | Index *pIdx; |
5188 | WhereScan scan; |
5189 | |
5190 | pWInfo = pBuilder->pWInfo; |
5191 | if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; |
5192 | assert( pWInfo->pTabList->nSrc>=1 ); |
5193 | pItem = pWInfo->pTabList->a; |
5194 | pTab = pItem->pTab; |
5195 | if( IsVirtual(pTab) ) return 0; |
5196 | if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){ |
5197 | testcase( pItem->fg.isIndexedBy ); |
5198 | testcase( pItem->fg.notIndexed ); |
5199 | return 0; |
5200 | } |
5201 | iCur = pItem->iCursor; |
5202 | pWC = &pWInfo->sWC; |
5203 | pLoop = pBuilder->pNew; |
5204 | pLoop->wsFlags = 0; |
5205 | pLoop->nSkip = 0; |
5206 | pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0); |
5207 | while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan); |
5208 | if( pTerm ){ |
5209 | testcase( pTerm->eOperator & WO_IS ); |
5210 | pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; |
5211 | pLoop->aLTerm[0] = pTerm; |
5212 | pLoop->nLTerm = 1; |
5213 | pLoop->u.btree.nEq = 1; |
5214 | /* TUNING: Cost of a rowid lookup is 10 */ |
5215 | pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ |
5216 | }else{ |
5217 | for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ |
5218 | int opMask; |
5219 | assert( pLoop->aLTermSpace==pLoop->aLTerm ); |
5220 | if( !IsUniqueIndex(pIdx) |
5221 | || pIdx->pPartIdxWhere!=0 |
5222 | || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) |
5223 | ) continue; |
5224 | opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; |
5225 | for(j=0; j<pIdx->nKeyCol; j++){ |
5226 | pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx); |
5227 | while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan); |
5228 | if( pTerm==0 ) break; |
5229 | testcase( pTerm->eOperator & WO_IS ); |
5230 | pLoop->aLTerm[j] = pTerm; |
5231 | } |
5232 | if( j!=pIdx->nKeyCol ) continue; |
5233 | pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; |
5234 | if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){ |
5235 | pLoop->wsFlags |= WHERE_IDX_ONLY; |
5236 | } |
5237 | pLoop->nLTerm = j; |
5238 | pLoop->u.btree.nEq = j; |
5239 | pLoop->u.btree.pIndex = pIdx; |
5240 | /* TUNING: Cost of a unique index lookup is 15 */ |
5241 | pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */ |
5242 | break; |
5243 | } |
5244 | } |
5245 | if( pLoop->wsFlags ){ |
5246 | pLoop->nOut = (LogEst)1; |
5247 | pWInfo->a[0].pWLoop = pLoop; |
5248 | assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] ); |
5249 | pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */ |
5250 | pWInfo->a[0].iTabCur = iCur; |
5251 | pWInfo->nRowOut = 1; |
5252 | if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; |
5253 | if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ |
5254 | pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; |
5255 | } |
5256 | if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS; |
5257 | #ifdef SQLITE_DEBUG |
5258 | pLoop->cId = '0'; |
5259 | #endif |
5260 | #ifdef WHERETRACE_ENABLED |
5261 | if( sqlite3WhereTrace ){ |
5262 | sqlite3DebugPrintf("whereShortCut() used to compute solution\n" ); |
5263 | } |
5264 | #endif |
5265 | return 1; |
5266 | } |
5267 | return 0; |
5268 | } |
5269 | |
5270 | /* |
5271 | ** Helper function for exprIsDeterministic(). |
5272 | */ |
5273 | static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){ |
5274 | if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){ |
5275 | pWalker->eCode = 0; |
5276 | return WRC_Abort; |
5277 | } |
5278 | return WRC_Continue; |
5279 | } |
5280 | |
5281 | /* |
5282 | ** Return true if the expression contains no non-deterministic SQL |
5283 | ** functions. Do not consider non-deterministic SQL functions that are |
5284 | ** part of sub-select statements. |
5285 | */ |
5286 | static int exprIsDeterministic(Expr *p){ |
5287 | Walker w; |
5288 | memset(&w, 0, sizeof(w)); |
5289 | w.eCode = 1; |
5290 | w.xExprCallback = exprNodeIsDeterministic; |
5291 | w.xSelectCallback = sqlite3SelectWalkFail; |
5292 | sqlite3WalkExpr(&w, p); |
5293 | return w.eCode; |
5294 | } |
5295 | |
5296 | |
5297 | #ifdef WHERETRACE_ENABLED |
5298 | /* |
5299 | ** Display all WhereLoops in pWInfo |
5300 | */ |
5301 | static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ |
5302 | if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ |
5303 | WhereLoop *p; |
5304 | int i; |
5305 | static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" |
5306 | "ABCDEFGHIJKLMNOPQRSTUVWYXZ" ; |
5307 | for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ |
5308 | p->cId = zLabel[i%(sizeof(zLabel)-1)]; |
5309 | sqlite3WhereLoopPrint(p, pWC); |
5310 | } |
5311 | } |
5312 | } |
5313 | # define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C) |
5314 | #else |
5315 | # define WHERETRACE_ALL_LOOPS(W,C) |
5316 | #endif |
5317 | |
5318 | /* Attempt to omit tables from a join that do not affect the result. |
5319 | ** For a table to not affect the result, the following must be true: |
5320 | ** |
5321 | ** 1) The query must not be an aggregate. |
5322 | ** 2) The table must be the RHS of a LEFT JOIN. |
5323 | ** 3) Either the query must be DISTINCT, or else the ON or USING clause |
5324 | ** must contain a constraint that limits the scan of the table to |
5325 | ** at most a single row. |
5326 | ** 4) The table must not be referenced by any part of the query apart |
5327 | ** from its own USING or ON clause. |
5328 | ** |
5329 | ** For example, given: |
5330 | ** |
5331 | ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1); |
5332 | ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2); |
5333 | ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3); |
5334 | ** |
5335 | ** then table t2 can be omitted from the following: |
5336 | ** |
5337 | ** SELECT v1, v3 FROM t1 |
5338 | ** LEFT JOIN t2 ON (t1.ipk=t2.ipk) |
5339 | ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) |
5340 | ** |
5341 | ** or from: |
5342 | ** |
5343 | ** SELECT DISTINCT v1, v3 FROM t1 |
5344 | ** LEFT JOIN t2 |
5345 | ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) |
5346 | */ |
5347 | static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( |
5348 | WhereInfo *pWInfo, |
5349 | Bitmask notReady |
5350 | ){ |
5351 | int i; |
5352 | Bitmask tabUsed; |
5353 | |
5354 | /* Preconditions checked by the caller */ |
5355 | assert( pWInfo->nLevel>=2 ); |
5356 | assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) ); |
5357 | |
5358 | /* These two preconditions checked by the caller combine to guarantee |
5359 | ** condition (1) of the header comment */ |
5360 | assert( pWInfo->pResultSet!=0 ); |
5361 | assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) ); |
5362 | |
5363 | tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet); |
5364 | if( pWInfo->pOrderBy ){ |
5365 | tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy); |
5366 | } |
5367 | for(i=pWInfo->nLevel-1; i>=1; i--){ |
5368 | WhereTerm *pTerm, *pEnd; |
5369 | SrcItem *pItem; |
5370 | WhereLoop *pLoop; |
5371 | pLoop = pWInfo->a[i].pWLoop; |
5372 | pItem = &pWInfo->pTabList->a[pLoop->iTab]; |
5373 | if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue; |
5374 | if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0 |
5375 | && (pLoop->wsFlags & WHERE_ONEROW)==0 |
5376 | ){ |
5377 | continue; |
5378 | } |
5379 | if( (tabUsed & pLoop->maskSelf)!=0 ) continue; |
5380 | pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm; |
5381 | for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ |
5382 | if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ |
5383 | if( !ExprHasProperty(pTerm->pExpr, EP_OuterON) |
5384 | || pTerm->pExpr->w.iJoin!=pItem->iCursor |
5385 | ){ |
5386 | break; |
5387 | } |
5388 | } |
5389 | } |
5390 | if( pTerm<pEnd ) continue; |
5391 | WHERETRACE(0xffff, ("-> drop loop %c not used\n" , pLoop->cId)); |
5392 | notReady &= ~pLoop->maskSelf; |
5393 | for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ |
5394 | if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ |
5395 | pTerm->wtFlags |= TERM_CODED; |
5396 | } |
5397 | } |
5398 | if( i!=pWInfo->nLevel-1 ){ |
5399 | int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel); |
5400 | memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte); |
5401 | } |
5402 | pWInfo->nLevel--; |
5403 | assert( pWInfo->nLevel>0 ); |
5404 | } |
5405 | return notReady; |
5406 | } |
5407 | |
5408 | /* |
5409 | ** Check to see if there are any SEARCH loops that might benefit from |
5410 | ** using a Bloom filter. Consider a Bloom filter if: |
5411 | ** |
5412 | ** (1) The SEARCH happens more than N times where N is the number |
5413 | ** of rows in the table that is being considered for the Bloom |
5414 | ** filter. |
5415 | ** (2) Some searches are expected to find zero rows. (This is determined |
5416 | ** by the WHERE_SELFCULL flag on the term.) |
5417 | ** (3) Bloom-filter processing is not disabled. (Checked by the |
5418 | ** caller.) |
5419 | ** (4) The size of the table being searched is known by ANALYZE. |
5420 | ** |
5421 | ** This block of code merely checks to see if a Bloom filter would be |
5422 | ** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the |
5423 | ** WhereLoop. The implementation of the Bloom filter comes further |
5424 | ** down where the code for each WhereLoop is generated. |
5425 | */ |
5426 | static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( |
5427 | const WhereInfo *pWInfo |
5428 | ){ |
5429 | int i; |
5430 | LogEst nSearch; |
5431 | |
5432 | assert( pWInfo->nLevel>=2 ); |
5433 | assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) ); |
5434 | nSearch = pWInfo->a[0].pWLoop->nOut; |
5435 | for(i=1; i<pWInfo->nLevel; i++){ |
5436 | WhereLoop *pLoop = pWInfo->a[i].pWLoop; |
5437 | const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ); |
5438 | if( (pLoop->wsFlags & reqFlags)==reqFlags |
5439 | /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ |
5440 | && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0) |
5441 | ){ |
5442 | SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; |
5443 | Table *pTab = pItem->pTab; |
5444 | pTab->tabFlags |= TF_StatsUsed; |
5445 | if( nSearch > pTab->nRowLogEst |
5446 | && (pTab->tabFlags & TF_HasStat1)!=0 |
5447 | ){ |
5448 | testcase( pItem->fg.jointype & JT_LEFT ); |
5449 | pLoop->wsFlags |= WHERE_BLOOMFILTER; |
5450 | pLoop->wsFlags &= ~WHERE_IDX_ONLY; |
5451 | WHERETRACE(0xffff, ( |
5452 | "-> use Bloom-filter on loop %c because there are ~%.1e " |
5453 | "lookups into %s which has only ~%.1e rows\n" , |
5454 | pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName, |
5455 | (double)sqlite3LogEstToInt(pTab->nRowLogEst))); |
5456 | } |
5457 | } |
5458 | nSearch += pLoop->nOut; |
5459 | } |
5460 | } |
5461 | |
5462 | /* |
5463 | ** This is an sqlite3ParserAddCleanup() callback that is invoked to |
5464 | ** free the Parse->pIdxExpr list when the Parse object is destroyed. |
5465 | */ |
5466 | static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ |
5467 | Parse *pParse = (Parse*)pObject; |
5468 | while( pParse->pIdxExpr!=0 ){ |
5469 | IndexedExpr *p = pParse->pIdxExpr; |
5470 | pParse->pIdxExpr = p->pIENext; |
5471 | sqlite3ExprDelete(db, p->pExpr); |
5472 | sqlite3DbFreeNN(db, p); |
5473 | } |
5474 | } |
5475 | |
5476 | /* |
5477 | ** The index pIdx is used by a query and contains one or more expressions. |
5478 | ** In other words pIdx is an index on an expression. iIdxCur is the cursor |
5479 | ** number for the index and iDataCur is the cursor number for the corresponding |
5480 | ** table. |
5481 | ** |
5482 | ** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for |
5483 | ** each of the expressions in the index so that the expression code generator |
5484 | ** will know to replace occurrences of the indexed expression with |
5485 | ** references to the corresponding column of the index. |
5486 | */ |
5487 | static SQLITE_NOINLINE void whereAddIndexedExpr( |
5488 | Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */ |
5489 | Index *pIdx, /* The index-on-expression that contains the expressions */ |
5490 | int iIdxCur, /* Cursor number for pIdx */ |
5491 | SrcItem *pTabItem /* The FROM clause entry for the table */ |
5492 | ){ |
5493 | int i; |
5494 | IndexedExpr *p; |
5495 | Table *pTab; |
5496 | assert( pIdx->bHasExpr ); |
5497 | pTab = pIdx->pTable; |
5498 | for(i=0; i<pIdx->nColumn; i++){ |
5499 | Expr *pExpr; |
5500 | int j = pIdx->aiColumn[i]; |
5501 | int bMaybeNullRow; |
5502 | if( j==XN_EXPR ){ |
5503 | pExpr = pIdx->aColExpr->a[i].pExpr; |
5504 | testcase( pTabItem->fg.jointype & JT_LEFT ); |
5505 | testcase( pTabItem->fg.jointype & JT_RIGHT ); |
5506 | testcase( pTabItem->fg.jointype & JT_LTORJ ); |
5507 | bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; |
5508 | }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ |
5509 | pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); |
5510 | bMaybeNullRow = 0; |
5511 | }else{ |
5512 | continue; |
5513 | } |
5514 | if( sqlite3ExprIsConstant(pExpr) ) continue; |
5515 | p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); |
5516 | if( p==0 ) break; |
5517 | p->pIENext = pParse->pIdxExpr; |
5518 | p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); |
5519 | p->iDataCur = pTabItem->iCursor; |
5520 | p->iIdxCur = iIdxCur; |
5521 | p->iIdxCol = i; |
5522 | p->bMaybeNullRow = bMaybeNullRow; |
5523 | #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS |
5524 | p->zIdxName = pIdx->zName; |
5525 | #endif |
5526 | pParse->pIdxExpr = p; |
5527 | if( p->pIENext==0 ){ |
5528 | sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); |
5529 | } |
5530 | } |
5531 | } |
5532 | |
5533 | /* |
5534 | ** Generate the beginning of the loop used for WHERE clause processing. |
5535 | ** The return value is a pointer to an opaque structure that contains |
5536 | ** information needed to terminate the loop. Later, the calling routine |
5537 | ** should invoke sqlite3WhereEnd() with the return value of this function |
5538 | ** in order to complete the WHERE clause processing. |
5539 | ** |
5540 | ** If an error occurs, this routine returns NULL. |
5541 | ** |
5542 | ** The basic idea is to do a nested loop, one loop for each table in |
5543 | ** the FROM clause of a select. (INSERT and UPDATE statements are the |
5544 | ** same as a SELECT with only a single table in the FROM clause.) For |
5545 | ** example, if the SQL is this: |
5546 | ** |
5547 | ** SELECT * FROM t1, t2, t3 WHERE ...; |
5548 | ** |
5549 | ** Then the code generated is conceptually like the following: |
5550 | ** |
5551 | ** foreach row1 in t1 do \ Code generated |
5552 | ** foreach row2 in t2 do |-- by sqlite3WhereBegin() |
5553 | ** foreach row3 in t3 do / |
5554 | ** ... |
5555 | ** end \ Code generated |
5556 | ** end |-- by sqlite3WhereEnd() |
5557 | ** end / |
5558 | ** |
5559 | ** Note that the loops might not be nested in the order in which they |
5560 | ** appear in the FROM clause if a different order is better able to make |
5561 | ** use of indices. Note also that when the IN operator appears in |
5562 | ** the WHERE clause, it might result in additional nested loops for |
5563 | ** scanning through all values on the right-hand side of the IN. |
5564 | ** |
5565 | ** There are Btree cursors associated with each table. t1 uses cursor |
5566 | ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. |
5567 | ** And so forth. This routine generates code to open those VDBE cursors |
5568 | ** and sqlite3WhereEnd() generates the code to close them. |
5569 | ** |
5570 | ** The code that sqlite3WhereBegin() generates leaves the cursors named |
5571 | ** in pTabList pointing at their appropriate entries. The [...] code |
5572 | ** can use OP_Column and OP_Rowid opcodes on these cursors to extract |
5573 | ** data from the various tables of the loop. |
5574 | ** |
5575 | ** If the WHERE clause is empty, the foreach loops must each scan their |
5576 | ** entire tables. Thus a three-way join is an O(N^3) operation. But if |
5577 | ** the tables have indices and there are terms in the WHERE clause that |
5578 | ** refer to those indices, a complete table scan can be avoided and the |
5579 | ** code will run much faster. Most of the work of this routine is checking |
5580 | ** to see if there are indices that can be used to speed up the loop. |
5581 | ** |
5582 | ** Terms of the WHERE clause are also used to limit which rows actually |
5583 | ** make it to the "..." in the middle of the loop. After each "foreach", |
5584 | ** terms of the WHERE clause that use only terms in that loop and outer |
5585 | ** loops are evaluated and if false a jump is made around all subsequent |
5586 | ** inner loops (or around the "..." if the test occurs within the inner- |
5587 | ** most loop) |
5588 | ** |
5589 | ** OUTER JOINS |
5590 | ** |
5591 | ** An outer join of tables t1 and t2 is conceptally coded as follows: |
5592 | ** |
5593 | ** foreach row1 in t1 do |
5594 | ** flag = 0 |
5595 | ** foreach row2 in t2 do |
5596 | ** start: |
5597 | ** ... |
5598 | ** flag = 1 |
5599 | ** end |
5600 | ** if flag==0 then |
5601 | ** move the row2 cursor to a null row |
5602 | ** goto start |
5603 | ** fi |
5604 | ** end |
5605 | ** |
5606 | ** ORDER BY CLAUSE PROCESSING |
5607 | ** |
5608 | ** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause |
5609 | ** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement |
5610 | ** if there is one. If there is no ORDER BY clause or if this routine |
5611 | ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. |
5612 | ** |
5613 | ** The iIdxCur parameter is the cursor number of an index. If |
5614 | ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index |
5615 | ** to use for OR clause processing. The WHERE clause should use this |
5616 | ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is |
5617 | ** the first cursor in an array of cursors for all indices. iIdxCur should |
5618 | ** be used to compute the appropriate cursor depending on which index is |
5619 | ** used. |
5620 | */ |
5621 | WhereInfo *sqlite3WhereBegin( |
5622 | Parse *pParse, /* The parser context */ |
5623 | SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ |
5624 | Expr *pWhere, /* The WHERE clause */ |
5625 | ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ |
5626 | ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ |
5627 | Select *pSelect, /* The entire SELECT statement */ |
5628 | u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ |
5629 | int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number |
5630 | ** If WHERE_USE_LIMIT, then the limit amount */ |
5631 | ){ |
5632 | int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ |
5633 | int nTabList; /* Number of elements in pTabList */ |
5634 | WhereInfo *pWInfo; /* Will become the return value of this function */ |
5635 | Vdbe *v = pParse->pVdbe; /* The virtual database engine */ |
5636 | Bitmask notReady; /* Cursors that are not yet positioned */ |
5637 | WhereLoopBuilder sWLB; /* The WhereLoop builder */ |
5638 | WhereMaskSet *pMaskSet; /* The expression mask set */ |
5639 | WhereLevel *pLevel; /* A single level in pWInfo->a[] */ |
5640 | WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ |
5641 | int ii; /* Loop counter */ |
5642 | sqlite3 *db; /* Database connection */ |
5643 | int rc; /* Return code */ |
5644 | u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */ |
5645 | |
5646 | assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( |
5647 | (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 |
5648 | && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 |
5649 | )); |
5650 | |
5651 | /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */ |
5652 | assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 |
5653 | || (wctrlFlags & WHERE_USE_LIMIT)==0 ); |
5654 | |
5655 | /* Variable initialization */ |
5656 | db = pParse->db; |
5657 | memset(&sWLB, 0, sizeof(sWLB)); |
5658 | |
5659 | /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ |
5660 | testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); |
5661 | if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; |
5662 | |
5663 | /* The number of tables in the FROM clause is limited by the number of |
5664 | ** bits in a Bitmask |
5665 | */ |
5666 | testcase( pTabList->nSrc==BMS ); |
5667 | if( pTabList->nSrc>BMS ){ |
5668 | sqlite3ErrorMsg(pParse, "at most %d tables in a join" , BMS); |
5669 | return 0; |
5670 | } |
5671 | |
5672 | /* This function normally generates a nested loop for all tables in |
5673 | ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should |
5674 | ** only generate code for the first table in pTabList and assume that |
5675 | ** any cursors associated with subsequent tables are uninitialized. |
5676 | */ |
5677 | nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc; |
5678 | |
5679 | /* Allocate and initialize the WhereInfo structure that will become the |
5680 | ** return value. A single allocation is used to store the WhereInfo |
5681 | ** struct, the contents of WhereInfo.a[], the WhereClause structure |
5682 | ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte |
5683 | ** field (type Bitmask) it must be aligned on an 8-byte boundary on |
5684 | ** some architectures. Hence the ROUND8() below. |
5685 | */ |
5686 | nByteWInfo = ROUND8P(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); |
5687 | pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); |
5688 | if( db->mallocFailed ){ |
5689 | sqlite3DbFree(db, pWInfo); |
5690 | pWInfo = 0; |
5691 | goto whereBeginError; |
5692 | } |
5693 | pWInfo->pParse = pParse; |
5694 | pWInfo->pTabList = pTabList; |
5695 | pWInfo->pOrderBy = pOrderBy; |
5696 | #if WHERETRACE_ENABLED |
5697 | pWInfo->pWhere = pWhere; |
5698 | #endif |
5699 | pWInfo->pResultSet = pResultSet; |
5700 | pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; |
5701 | pWInfo->nLevel = nTabList; |
5702 | pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse); |
5703 | pWInfo->wctrlFlags = wctrlFlags; |
5704 | pWInfo->iLimit = iAuxArg; |
5705 | pWInfo->savedNQueryLoop = pParse->nQueryLoop; |
5706 | pWInfo->pSelect = pSelect; |
5707 | memset(&pWInfo->nOBSat, 0, |
5708 | offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); |
5709 | memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); |
5710 | assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ |
5711 | pMaskSet = &pWInfo->sMaskSet; |
5712 | pMaskSet->n = 0; |
5713 | pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be |
5714 | ** a valid cursor number, to avoid an initial |
5715 | ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */ |
5716 | sWLB.pWInfo = pWInfo; |
5717 | sWLB.pWC = &pWInfo->sWC; |
5718 | sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); |
5719 | assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) ); |
5720 | whereLoopInit(sWLB.pNew); |
5721 | #ifdef SQLITE_DEBUG |
5722 | sWLB.pNew->cId = '*'; |
5723 | #endif |
5724 | |
5725 | /* Split the WHERE clause into separate subexpressions where each |
5726 | ** subexpression is separated by an AND operator. |
5727 | */ |
5728 | sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); |
5729 | sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); |
5730 | |
5731 | /* Special case: No FROM clause |
5732 | */ |
5733 | if( nTabList==0 ){ |
5734 | if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; |
5735 | if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0 |
5736 | && OptimizationEnabled(db, SQLITE_DistinctOpt) |
5737 | ){ |
5738 | pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; |
5739 | } |
5740 | ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW" )); |
5741 | }else{ |
5742 | /* Assign a bit from the bitmask to every term in the FROM clause. |
5743 | ** |
5744 | ** The N-th term of the FROM clause is assigned a bitmask of 1<<N. |
5745 | ** |
5746 | ** The rule of the previous sentence ensures thta if X is the bitmask for |
5747 | ** a table T, then X-1 is the bitmask for all other tables to the left of T. |
5748 | ** Knowing the bitmask for all tables to the left of a left join is |
5749 | ** important. Ticket #3015. |
5750 | ** |
5751 | ** Note that bitmasks are created for all pTabList->nSrc tables in |
5752 | ** pTabList, not just the first nTabList tables. nTabList is normally |
5753 | ** equal to pTabList->nSrc but might be shortened to 1 if the |
5754 | ** WHERE_OR_SUBCLAUSE flag is set. |
5755 | */ |
5756 | ii = 0; |
5757 | do{ |
5758 | createMask(pMaskSet, pTabList->a[ii].iCursor); |
5759 | sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); |
5760 | }while( (++ii)<pTabList->nSrc ); |
5761 | #ifdef SQLITE_DEBUG |
5762 | { |
5763 | Bitmask mx = 0; |
5764 | for(ii=0; ii<pTabList->nSrc; ii++){ |
5765 | Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); |
5766 | assert( m>=mx ); |
5767 | mx = m; |
5768 | } |
5769 | } |
5770 | #endif |
5771 | } |
5772 | |
5773 | /* Analyze all of the subexpressions. */ |
5774 | sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); |
5775 | if( pSelect && pSelect->pLimit ){ |
5776 | sqlite3WhereAddLimit(&pWInfo->sWC, pSelect); |
5777 | } |
5778 | if( pParse->nErr ) goto whereBeginError; |
5779 | |
5780 | /* Special case: WHERE terms that do not refer to any tables in the join |
5781 | ** (constant expressions). Evaluate each such term, and jump over all the |
5782 | ** generated code if the result is not true. |
5783 | ** |
5784 | ** Do not do this if the expression contains non-deterministic functions |
5785 | ** that are not within a sub-select. This is not strictly required, but |
5786 | ** preserves SQLite's legacy behaviour in the following two cases: |
5787 | ** |
5788 | ** FROM ... WHERE random()>0; -- eval random() once per row |
5789 | ** FROM ... WHERE (SELECT random())>0; -- eval random() once overall |
5790 | */ |
5791 | for(ii=0; ii<sWLB.pWC->nBase; ii++){ |
5792 | WhereTerm *pT = &sWLB.pWC->a[ii]; |
5793 | if( pT->wtFlags & TERM_VIRTUAL ) continue; |
5794 | if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ |
5795 | sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); |
5796 | pT->wtFlags |= TERM_CODED; |
5797 | } |
5798 | } |
5799 | |
5800 | if( wctrlFlags & WHERE_WANT_DISTINCT ){ |
5801 | if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ |
5802 | /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via |
5803 | ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ |
5804 | wctrlFlags &= ~WHERE_WANT_DISTINCT; |
5805 | pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT; |
5806 | }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ |
5807 | /* The DISTINCT marking is pointless. Ignore it. */ |
5808 | pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; |
5809 | }else if( pOrderBy==0 ){ |
5810 | /* Try to ORDER BY the result set to make distinct processing easier */ |
5811 | pWInfo->wctrlFlags |= WHERE_DISTINCTBY; |
5812 | pWInfo->pOrderBy = pResultSet; |
5813 | } |
5814 | } |
5815 | |
5816 | /* Construct the WhereLoop objects */ |
5817 | #if defined(WHERETRACE_ENABLED) |
5818 | if( sqlite3WhereTrace & 0xffff ){ |
5819 | sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x" ,wctrlFlags); |
5820 | if( wctrlFlags & WHERE_USE_LIMIT ){ |
5821 | sqlite3DebugPrintf(", limit: %d" , iAuxArg); |
5822 | } |
5823 | sqlite3DebugPrintf(")\n" ); |
5824 | if( sqlite3WhereTrace & 0x100 ){ |
5825 | Select sSelect; |
5826 | memset(&sSelect, 0, sizeof(sSelect)); |
5827 | sSelect.selFlags = SF_WhereBegin; |
5828 | sSelect.pSrc = pTabList; |
5829 | sSelect.pWhere = pWhere; |
5830 | sSelect.pOrderBy = pOrderBy; |
5831 | sSelect.pEList = pResultSet; |
5832 | sqlite3TreeViewSelect(0, &sSelect, 0); |
5833 | } |
5834 | } |
5835 | if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ |
5836 | sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n" ); |
5837 | sqlite3WhereClausePrint(sWLB.pWC); |
5838 | } |
5839 | #endif |
5840 | |
5841 | if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ |
5842 | rc = whereLoopAddAll(&sWLB); |
5843 | if( rc ) goto whereBeginError; |
5844 | |
5845 | #ifdef SQLITE_ENABLE_STAT4 |
5846 | /* If one or more WhereTerm.truthProb values were used in estimating |
5847 | ** loop parameters, but then those truthProb values were subsequently |
5848 | ** changed based on STAT4 information while computing subsequent loops, |
5849 | ** then we need to rerun the whole loop building process so that all |
5850 | ** loops will be built using the revised truthProb values. */ |
5851 | if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ |
5852 | WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); |
5853 | WHERETRACE(0xffff, |
5854 | ("**** Redo all loop computations due to" |
5855 | " TERM_HIGHTRUTH changes ****\n" )); |
5856 | while( pWInfo->pLoops ){ |
5857 | WhereLoop *p = pWInfo->pLoops; |
5858 | pWInfo->pLoops = p->pNextLoop; |
5859 | whereLoopDelete(db, p); |
5860 | } |
5861 | rc = whereLoopAddAll(&sWLB); |
5862 | if( rc ) goto whereBeginError; |
5863 | } |
5864 | #endif |
5865 | WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); |
5866 | |
5867 | wherePathSolver(pWInfo, 0); |
5868 | if( db->mallocFailed ) goto whereBeginError; |
5869 | if( pWInfo->pOrderBy ){ |
5870 | wherePathSolver(pWInfo, pWInfo->nRowOut+1); |
5871 | if( db->mallocFailed ) goto whereBeginError; |
5872 | } |
5873 | } |
5874 | if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ |
5875 | pWInfo->revMask = ALLBITS; |
5876 | } |
5877 | if( pParse->nErr ){ |
5878 | goto whereBeginError; |
5879 | } |
5880 | assert( db->mallocFailed==0 ); |
5881 | #ifdef WHERETRACE_ENABLED |
5882 | if( sqlite3WhereTrace ){ |
5883 | sqlite3DebugPrintf("---- Solution nRow=%d" , pWInfo->nRowOut); |
5884 | if( pWInfo->nOBSat>0 ){ |
5885 | sqlite3DebugPrintf(" ORDERBY=%d,0x%llx" , pWInfo->nOBSat, pWInfo->revMask); |
5886 | } |
5887 | switch( pWInfo->eDistinct ){ |
5888 | case WHERE_DISTINCT_UNIQUE: { |
5889 | sqlite3DebugPrintf(" DISTINCT=unique" ); |
5890 | break; |
5891 | } |
5892 | case WHERE_DISTINCT_ORDERED: { |
5893 | sqlite3DebugPrintf(" DISTINCT=ordered" ); |
5894 | break; |
5895 | } |
5896 | case WHERE_DISTINCT_UNORDERED: { |
5897 | sqlite3DebugPrintf(" DISTINCT=unordered" ); |
5898 | break; |
5899 | } |
5900 | } |
5901 | sqlite3DebugPrintf("\n" ); |
5902 | for(ii=0; ii<pWInfo->nLevel; ii++){ |
5903 | sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); |
5904 | } |
5905 | } |
5906 | #endif |
5907 | |
5908 | /* Attempt to omit tables from a join that do not affect the result. |
5909 | ** See the comment on whereOmitNoopJoin() for further information. |
5910 | ** |
5911 | ** This query optimization is factored out into a separate "no-inline" |
5912 | ** procedure to keep the sqlite3WhereBegin() procedure from becoming |
5913 | ** too large. If sqlite3WhereBegin() becomes too large, that prevents |
5914 | ** some C-compiler optimizers from in-lining the |
5915 | ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to |
5916 | ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. |
5917 | */ |
5918 | notReady = ~(Bitmask)0; |
5919 | if( pWInfo->nLevel>=2 |
5920 | && pResultSet!=0 /* these two combine to guarantee */ |
5921 | && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ |
5922 | && OptimizationEnabled(db, SQLITE_OmitNoopJoin) |
5923 | ){ |
5924 | notReady = whereOmitNoopJoin(pWInfo, notReady); |
5925 | nTabList = pWInfo->nLevel; |
5926 | assert( nTabList>0 ); |
5927 | } |
5928 | |
5929 | /* Check to see if there are any SEARCH loops that might benefit from |
5930 | ** using a Bloom filter. |
5931 | */ |
5932 | if( pWInfo->nLevel>=2 |
5933 | && OptimizationEnabled(db, SQLITE_BloomFilter) |
5934 | ){ |
5935 | whereCheckIfBloomFilterIsUseful(pWInfo); |
5936 | } |
5937 | |
5938 | #if defined(WHERETRACE_ENABLED) |
5939 | if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ |
5940 | sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n" ); |
5941 | sqlite3WhereClausePrint(sWLB.pWC); |
5942 | } |
5943 | WHERETRACE(0xffff,("*** Optimizer Finished ***\n" )); |
5944 | #endif |
5945 | pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; |
5946 | |
5947 | /* If the caller is an UPDATE or DELETE statement that is requesting |
5948 | ** to use a one-pass algorithm, determine if this is appropriate. |
5949 | ** |
5950 | ** A one-pass approach can be used if the caller has requested one |
5951 | ** and either (a) the scan visits at most one row or (b) each |
5952 | ** of the following are true: |
5953 | ** |
5954 | ** * the caller has indicated that a one-pass approach can be used |
5955 | ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and |
5956 | ** * the table is not a virtual table, and |
5957 | ** * either the scan does not use the OR optimization or the caller |
5958 | ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified |
5959 | ** for DELETE). |
5960 | ** |
5961 | ** The last qualification is because an UPDATE statement uses |
5962 | ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can |
5963 | ** use a one-pass approach, and this is not set accurately for scans |
5964 | ** that use the OR optimization. |
5965 | */ |
5966 | assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); |
5967 | if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ |
5968 | int wsFlags = pWInfo->a[0].pWLoop->wsFlags; |
5969 | int bOnerow = (wsFlags & WHERE_ONEROW)!=0; |
5970 | assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) ); |
5971 | if( bOnerow || ( |
5972 | 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) |
5973 | && !IsVirtual(pTabList->a[0].pTab) |
5974 | && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) |
5975 | )){ |
5976 | pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; |
5977 | if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ |
5978 | if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ |
5979 | bFordelete = OPFLAG_FORDELETE; |
5980 | } |
5981 | pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY); |
5982 | } |
5983 | } |
5984 | } |
5985 | |
5986 | /* Open all tables in the pTabList and any indices selected for |
5987 | ** searching those tables. |
5988 | */ |
5989 | for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){ |
5990 | Table *pTab; /* Table to open */ |
5991 | int iDb; /* Index of database containing table/index */ |
5992 | SrcItem *pTabItem; |
5993 | |
5994 | pTabItem = &pTabList->a[pLevel->iFrom]; |
5995 | pTab = pTabItem->pTab; |
5996 | iDb = sqlite3SchemaToIndex(db, pTab->pSchema); |
5997 | pLoop = pLevel->pWLoop; |
5998 | if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ |
5999 | /* Do nothing */ |
6000 | }else |
6001 | #ifndef SQLITE_OMIT_VIRTUALTABLE |
6002 | if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ |
6003 | const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); |
6004 | int iCur = pTabItem->iCursor; |
6005 | sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); |
6006 | }else if( IsVirtual(pTab) ){ |
6007 | /* noop */ |
6008 | }else |
6009 | #endif |
6010 | if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0 |
6011 | && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0) |
6012 | || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0 |
6013 | ){ |
6014 | int op = OP_OpenRead; |
6015 | if( pWInfo->eOnePass!=ONEPASS_OFF ){ |
6016 | op = OP_OpenWrite; |
6017 | pWInfo->aiCurOnePass[0] = pTabItem->iCursor; |
6018 | }; |
6019 | sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); |
6020 | assert( pTabItem->iCursor==pLevel->iTabCur ); |
6021 | testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); |
6022 | testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); |
6023 | if( pWInfo->eOnePass==ONEPASS_OFF |
6024 | && pTab->nCol<BMS |
6025 | && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0 |
6026 | && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0 |
6027 | ){ |
6028 | /* If we know that only a prefix of the record will be used, |
6029 | ** it is advantageous to reduce the "column count" field in |
6030 | ** the P4 operand of the OP_OpenRead/Write opcode. */ |
6031 | Bitmask b = pTabItem->colUsed; |
6032 | int n = 0; |
6033 | for(; b; b=b>>1, n++){} |
6034 | sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); |
6035 | assert( n<=pTab->nCol ); |
6036 | } |
6037 | #ifdef SQLITE_ENABLE_CURSOR_HINTS |
6038 | if( pLoop->u.btree.pIndex!=0 ){ |
6039 | sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); |
6040 | }else |
6041 | #endif |
6042 | { |
6043 | sqlite3VdbeChangeP5(v, bFordelete); |
6044 | } |
6045 | #ifdef SQLITE_ENABLE_COLUMN_USED_MASK |
6046 | sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, |
6047 | (const u8*)&pTabItem->colUsed, P4_INT64); |
6048 | #endif |
6049 | }else{ |
6050 | sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); |
6051 | } |
6052 | if( pLoop->wsFlags & WHERE_INDEXED ){ |
6053 | Index *pIx = pLoop->u.btree.pIndex; |
6054 | int iIndexCur; |
6055 | int op = OP_OpenRead; |
6056 | /* iAuxArg is always set to a positive value if ONEPASS is possible */ |
6057 | assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); |
6058 | if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) |
6059 | && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 |
6060 | ){ |
6061 | /* This is one term of an OR-optimization using the PRIMARY KEY of a |
6062 | ** WITHOUT ROWID table. No need for a separate index */ |
6063 | iIndexCur = pLevel->iTabCur; |
6064 | op = 0; |
6065 | }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ |
6066 | Index *pJ = pTabItem->pTab->pIndex; |
6067 | iIndexCur = iAuxArg; |
6068 | assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); |
6069 | while( ALWAYS(pJ) && pJ!=pIx ){ |
6070 | iIndexCur++; |
6071 | pJ = pJ->pNext; |
6072 | } |
6073 | op = OP_OpenWrite; |
6074 | pWInfo->aiCurOnePass[1] = iIndexCur; |
6075 | }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){ |
6076 | iIndexCur = iAuxArg; |
6077 | op = OP_ReopenIdx; |
6078 | }else{ |
6079 | iIndexCur = pParse->nTab++; |
6080 | if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){ |
6081 | whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); |
6082 | } |
6083 | } |
6084 | pLevel->iIdxCur = iIndexCur; |
6085 | assert( pIx!=0 ); |
6086 | assert( pIx->pSchema==pTab->pSchema ); |
6087 | assert( iIndexCur>=0 ); |
6088 | if( op ){ |
6089 | sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); |
6090 | sqlite3VdbeSetP4KeyInfo(pParse, pIx); |
6091 | if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 |
6092 | && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 |
6093 | && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0 |
6094 | && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 |
6095 | && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 |
6096 | && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED |
6097 | ){ |
6098 | sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); |
6099 | } |
6100 | VdbeComment((v, "%s" , pIx->zName)); |
6101 | #ifdef SQLITE_ENABLE_COLUMN_USED_MASK |
6102 | { |
6103 | u64 colUsed = 0; |
6104 | int ii, jj; |
6105 | for(ii=0; ii<pIx->nColumn; ii++){ |
6106 | jj = pIx->aiColumn[ii]; |
6107 | if( jj<0 ) continue; |
6108 | if( jj>63 ) jj = 63; |
6109 | if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue; |
6110 | colUsed |= ((u64)1)<<(ii<63 ? ii : 63); |
6111 | } |
6112 | sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0, |
6113 | (u8*)&colUsed, P4_INT64); |
6114 | } |
6115 | #endif /* SQLITE_ENABLE_COLUMN_USED_MASK */ |
6116 | } |
6117 | } |
6118 | if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); |
6119 | if( (pTabItem->fg.jointype & JT_RIGHT)!=0 |
6120 | && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0 |
6121 | ){ |
6122 | WhereRightJoin *pRJ = pLevel->pRJ; |
6123 | pRJ->iMatch = pParse->nTab++; |
6124 | pRJ->regBloom = ++pParse->nMem; |
6125 | sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom); |
6126 | pRJ->regReturn = ++pParse->nMem; |
6127 | sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn); |
6128 | assert( pTab==pTabItem->pTab ); |
6129 | if( HasRowid(pTab) ){ |
6130 | KeyInfo *pInfo; |
6131 | sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1); |
6132 | pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0); |
6133 | if( pInfo ){ |
6134 | pInfo->aColl[0] = 0; |
6135 | pInfo->aSortFlags[0] = 0; |
6136 | sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO); |
6137 | } |
6138 | }else{ |
6139 | Index *pPk = sqlite3PrimaryKeyIndex(pTab); |
6140 | sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol); |
6141 | sqlite3VdbeSetP4KeyInfo(pParse, pPk); |
6142 | } |
6143 | pLoop->wsFlags &= ~WHERE_IDX_ONLY; |
6144 | /* The nature of RIGHT JOIN processing is such that it messes up |
6145 | ** the output order. So omit any ORDER BY/GROUP BY elimination |
6146 | ** optimizations. We need to do an actual sort for RIGHT JOIN. */ |
6147 | pWInfo->nOBSat = 0; |
6148 | pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED; |
6149 | } |
6150 | } |
6151 | pWInfo->iTop = sqlite3VdbeCurrentAddr(v); |
6152 | if( db->mallocFailed ) goto whereBeginError; |
6153 | |
6154 | /* Generate the code to do the search. Each iteration of the for |
6155 | ** loop below generates code for a single nested loop of the VM |
6156 | ** program. |
6157 | */ |
6158 | for(ii=0; ii<nTabList; ii++){ |
6159 | int addrExplain; |
6160 | int wsFlags; |
6161 | SrcItem *pSrc; |
6162 | if( pParse->nErr ) goto whereBeginError; |
6163 | pLevel = &pWInfo->a[ii]; |
6164 | wsFlags = pLevel->pWLoop->wsFlags; |
6165 | pSrc = &pTabList->a[pLevel->iFrom]; |
6166 | if( pSrc->fg.isMaterialized ){ |
6167 | if( pSrc->fg.isCorrelated ){ |
6168 | sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); |
6169 | }else{ |
6170 | int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); |
6171 | sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); |
6172 | sqlite3VdbeJumpHere(v, iOnce); |
6173 | } |
6174 | } |
6175 | if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ |
6176 | if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){ |
6177 | #ifndef SQLITE_OMIT_AUTOMATIC_INDEX |
6178 | constructAutomaticIndex(pParse, &pWInfo->sWC, |
6179 | &pTabList->a[pLevel->iFrom], notReady, pLevel); |
6180 | #endif |
6181 | }else{ |
6182 | sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady); |
6183 | } |
6184 | if( db->mallocFailed ) goto whereBeginError; |
6185 | } |
6186 | addrExplain = sqlite3WhereExplainOneScan( |
6187 | pParse, pTabList, pLevel, wctrlFlags |
6188 | ); |
6189 | pLevel->addrBody = sqlite3VdbeCurrentAddr(v); |
6190 | notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady); |
6191 | pWInfo->iContinue = pLevel->addrCont; |
6192 | if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){ |
6193 | sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); |
6194 | } |
6195 | } |
6196 | |
6197 | /* Done. */ |
6198 | VdbeModuleComment((v, "Begin WHERE-core" )); |
6199 | pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v); |
6200 | return pWInfo; |
6201 | |
6202 | /* Jump here if malloc fails */ |
6203 | whereBeginError: |
6204 | if( pWInfo ){ |
6205 | pParse->nQueryLoop = pWInfo->savedNQueryLoop; |
6206 | whereInfoFree(db, pWInfo); |
6207 | } |
6208 | return 0; |
6209 | } |
6210 | |
6211 | /* |
6212 | ** Part of sqlite3WhereEnd() will rewrite opcodes to reference the |
6213 | ** index rather than the main table. In SQLITE_DEBUG mode, we want |
6214 | ** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine |
6215 | ** does that. |
6216 | */ |
6217 | #ifndef SQLITE_DEBUG |
6218 | # define OpcodeRewriteTrace(D,K,P) /* no-op */ |
6219 | #else |
6220 | # define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P) |
6221 | static void sqlite3WhereOpcodeRewriteTrace( |
6222 | sqlite3 *db, |
6223 | int pc, |
6224 | VdbeOp *pOp |
6225 | ){ |
6226 | if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return; |
6227 | sqlite3VdbePrintOp(0, pc, pOp); |
6228 | } |
6229 | #endif |
6230 | |
6231 | #ifdef SQLITE_DEBUG |
6232 | /* |
6233 | ** Return true if cursor iCur is opened by instruction k of the |
6234 | ** bytecode. Used inside of assert() only. |
6235 | */ |
6236 | static int cursorIsOpen(Vdbe *v, int iCur, int k){ |
6237 | while( k>=0 ){ |
6238 | VdbeOp *pOp = sqlite3VdbeGetOp(v,k--); |
6239 | if( pOp->p1!=iCur ) continue; |
6240 | if( pOp->opcode==OP_Close ) return 0; |
6241 | if( pOp->opcode==OP_OpenRead ) return 1; |
6242 | if( pOp->opcode==OP_OpenWrite ) return 1; |
6243 | if( pOp->opcode==OP_OpenDup ) return 1; |
6244 | if( pOp->opcode==OP_OpenAutoindex ) return 1; |
6245 | if( pOp->opcode==OP_OpenEphemeral ) return 1; |
6246 | } |
6247 | return 0; |
6248 | } |
6249 | #endif /* SQLITE_DEBUG */ |
6250 | |
6251 | /* |
6252 | ** Generate the end of the WHERE loop. See comments on |
6253 | ** sqlite3WhereBegin() for additional information. |
6254 | */ |
6255 | void sqlite3WhereEnd(WhereInfo *pWInfo){ |
6256 | Parse *pParse = pWInfo->pParse; |
6257 | Vdbe *v = pParse->pVdbe; |
6258 | int i; |
6259 | WhereLevel *pLevel; |
6260 | WhereLoop *pLoop; |
6261 | SrcList *pTabList = pWInfo->pTabList; |
6262 | sqlite3 *db = pParse->db; |
6263 | int iEnd = sqlite3VdbeCurrentAddr(v); |
6264 | int nRJ = 0; |
6265 | |
6266 | /* Generate loop termination code. |
6267 | */ |
6268 | VdbeModuleComment((v, "End WHERE-core" )); |
6269 | for(i=pWInfo->nLevel-1; i>=0; i--){ |
6270 | int addr; |
6271 | pLevel = &pWInfo->a[i]; |
6272 | if( pLevel->pRJ ){ |
6273 | /* Terminate the subroutine that forms the interior of the loop of |
6274 | ** the RIGHT JOIN table */ |
6275 | WhereRightJoin *pRJ = pLevel->pRJ; |
6276 | sqlite3VdbeResolveLabel(v, pLevel->addrCont); |
6277 | pLevel->addrCont = 0; |
6278 | pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v); |
6279 | sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1); |
6280 | VdbeCoverage(v); |
6281 | nRJ++; |
6282 | } |
6283 | pLoop = pLevel->pWLoop; |
6284 | if( pLevel->op!=OP_Noop ){ |
6285 | #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT |
6286 | int addrSeek = 0; |
6287 | Index *pIdx; |
6288 | int n; |
6289 | if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED |
6290 | && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */ |
6291 | && (pLoop->wsFlags & WHERE_INDEXED)!=0 |
6292 | && (pIdx = pLoop->u.btree.pIndex)->hasStat1 |
6293 | && (n = pLoop->u.btree.nDistinctCol)>0 |
6294 | && pIdx->aiRowLogEst[n]>=36 |
6295 | ){ |
6296 | int r1 = pParse->nMem+1; |
6297 | int j, op; |
6298 | for(j=0; j<n; j++){ |
6299 | sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j); |
6300 | } |
6301 | pParse->nMem += n+1; |
6302 | op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT; |
6303 | addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n); |
6304 | VdbeCoverageIf(v, op==OP_SeekLT); |
6305 | VdbeCoverageIf(v, op==OP_SeekGT); |
6306 | sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); |
6307 | } |
6308 | #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ |
6309 | /* The common case: Advance to the next row */ |
6310 | if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); |
6311 | sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); |
6312 | sqlite3VdbeChangeP5(v, pLevel->p5); |
6313 | VdbeCoverage(v); |
6314 | VdbeCoverageIf(v, pLevel->op==OP_Next); |
6315 | VdbeCoverageIf(v, pLevel->op==OP_Prev); |
6316 | VdbeCoverageIf(v, pLevel->op==OP_VNext); |
6317 | if( pLevel->regBignull ){ |
6318 | sqlite3VdbeResolveLabel(v, pLevel->addrBignull); |
6319 | sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1); |
6320 | VdbeCoverage(v); |
6321 | } |
6322 | #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT |
6323 | if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); |
6324 | #endif |
6325 | }else if( pLevel->addrCont ){ |
6326 | sqlite3VdbeResolveLabel(v, pLevel->addrCont); |
6327 | } |
6328 | if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){ |
6329 | struct InLoop *pIn; |
6330 | int j; |
6331 | sqlite3VdbeResolveLabel(v, pLevel->addrNxt); |
6332 | for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ |
6333 | assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull |
6334 | || pParse->db->mallocFailed ); |
6335 | sqlite3VdbeJumpHere(v, pIn->addrInTop+1); |
6336 | if( pIn->eEndLoopOp!=OP_Noop ){ |
6337 | if( pIn->nPrefix ){ |
6338 | int bEarlyOut = |
6339 | (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 |
6340 | && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0; |
6341 | if( pLevel->iLeftJoin ){ |
6342 | /* For LEFT JOIN queries, cursor pIn->iCur may not have been |
6343 | ** opened yet. This occurs for WHERE clauses such as |
6344 | ** "a = ? AND b IN (...)", where the index is on (a, b). If |
6345 | ** the RHS of the (a=?) is NULL, then the "b IN (...)" may |
6346 | ** never have been coded, but the body of the loop run to |
6347 | ** return the null-row. So, if the cursor is not open yet, |
6348 | ** jump over the OP_Next or OP_Prev instruction about to |
6349 | ** be coded. */ |
6350 | sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur, |
6351 | sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut); |
6352 | VdbeCoverage(v); |
6353 | } |
6354 | if( bEarlyOut ){ |
6355 | sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur, |
6356 | sqlite3VdbeCurrentAddr(v)+2, |
6357 | pIn->iBase, pIn->nPrefix); |
6358 | VdbeCoverage(v); |
6359 | /* Retarget the OP_IsNull against the left operand of IN so |
6360 | ** it jumps past the OP_IfNoHope. This is because the |
6361 | ** OP_IsNull also bypasses the OP_Affinity opcode that is |
6362 | ** required by OP_IfNoHope. */ |
6363 | sqlite3VdbeJumpHere(v, pIn->addrInTop+1); |
6364 | } |
6365 | } |
6366 | sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); |
6367 | VdbeCoverage(v); |
6368 | VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev); |
6369 | VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next); |
6370 | } |
6371 | sqlite3VdbeJumpHere(v, pIn->addrInTop-1); |
6372 | } |
6373 | } |
6374 | sqlite3VdbeResolveLabel(v, pLevel->addrBrk); |
6375 | if( pLevel->pRJ ){ |
6376 | sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1); |
6377 | VdbeCoverage(v); |
6378 | } |
6379 | if( pLevel->addrSkip ){ |
6380 | sqlite3VdbeGoto(v, pLevel->addrSkip); |
6381 | VdbeComment((v, "next skip-scan on %s" , pLoop->u.btree.pIndex->zName)); |
6382 | sqlite3VdbeJumpHere(v, pLevel->addrSkip); |
6383 | sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); |
6384 | } |
6385 | #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS |
6386 | if( pLevel->addrLikeRep ){ |
6387 | sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1), |
6388 | pLevel->addrLikeRep); |
6389 | VdbeCoverage(v); |
6390 | } |
6391 | #endif |
6392 | if( pLevel->iLeftJoin ){ |
6393 | int ws = pLoop->wsFlags; |
6394 | addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); |
6395 | assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); |
6396 | if( (ws & WHERE_IDX_ONLY)==0 ){ |
6397 | assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); |
6398 | sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); |
6399 | } |
6400 | if( (ws & WHERE_INDEXED) |
6401 | || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx) |
6402 | ){ |
6403 | if( ws & WHERE_MULTI_OR ){ |
6404 | Index *pIx = pLevel->u.pCoveringIdx; |
6405 | int iDb = sqlite3SchemaToIndex(db, pIx->pSchema); |
6406 | sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb); |
6407 | sqlite3VdbeSetP4KeyInfo(pParse, pIx); |
6408 | } |
6409 | sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); |
6410 | } |
6411 | if( pLevel->op==OP_Return ){ |
6412 | sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); |
6413 | }else{ |
6414 | sqlite3VdbeGoto(v, pLevel->addrFirst); |
6415 | } |
6416 | sqlite3VdbeJumpHere(v, addr); |
6417 | } |
6418 | VdbeModuleComment((v, "End WHERE-loop%d: %s" , i, |
6419 | pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); |
6420 | } |
6421 | |
6422 | assert( pWInfo->nLevel<=pTabList->nSrc ); |
6423 | for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){ |
6424 | int k, last; |
6425 | VdbeOp *pOp, *pLastOp; |
6426 | Index *pIdx = 0; |
6427 | SrcItem *pTabItem = &pTabList->a[pLevel->iFrom]; |
6428 | Table *pTab = pTabItem->pTab; |
6429 | assert( pTab!=0 ); |
6430 | pLoop = pLevel->pWLoop; |
6431 | |
6432 | /* Do RIGHT JOIN processing. Generate code that will output the |
6433 | ** unmatched rows of the right operand of the RIGHT JOIN with |
6434 | ** all of the columns of the left operand set to NULL. |
6435 | */ |
6436 | if( pLevel->pRJ ){ |
6437 | sqlite3WhereRightJoinLoop(pWInfo, i, pLevel); |
6438 | continue; |
6439 | } |
6440 | |
6441 | /* For a co-routine, change all OP_Column references to the table of |
6442 | ** the co-routine into OP_Copy of result contained in a register. |
6443 | ** OP_Rowid becomes OP_Null. |
6444 | */ |
6445 | if( pTabItem->fg.viaCoroutine ){ |
6446 | testcase( pParse->db->mallocFailed ); |
6447 | translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, |
6448 | pTabItem->regResult, 0); |
6449 | continue; |
6450 | } |
6451 | |
6452 | /* If this scan uses an index, make VDBE code substitutions to read data |
6453 | ** from the index instead of from the table where possible. In some cases |
6454 | ** this optimization prevents the table from ever being read, which can |
6455 | ** yield a significant performance boost. |
6456 | ** |
6457 | ** Calls to the code generator in between sqlite3WhereBegin and |
6458 | ** sqlite3WhereEnd will have created code that references the table |
6459 | ** directly. This loop scans all that code looking for opcodes |
6460 | ** that reference the table and converts them into opcodes that |
6461 | ** reference the index. |
6462 | */ |
6463 | if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ |
6464 | pIdx = pLoop->u.btree.pIndex; |
6465 | }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ |
6466 | pIdx = pLevel->u.pCoveringIdx; |
6467 | } |
6468 | if( pIdx |
6469 | && !db->mallocFailed |
6470 | ){ |
6471 | if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){ |
6472 | last = iEnd; |
6473 | }else{ |
6474 | last = pWInfo->iEndWhere; |
6475 | } |
6476 | if( pIdx->bHasExpr ){ |
6477 | IndexedExpr *p = pParse->pIdxExpr; |
6478 | while( p ){ |
6479 | if( p->iIdxCur==pLevel->iIdxCur ){ |
6480 | p->iDataCur = -1; |
6481 | p->iIdxCur = -1; |
6482 | } |
6483 | p = p->pIENext; |
6484 | } |
6485 | } |
6486 | k = pLevel->addrBody + 1; |
6487 | #ifdef SQLITE_DEBUG |
6488 | if( db->flags & SQLITE_VdbeAddopTrace ){ |
6489 | printf("TRANSLATE opcodes in range %d..%d\n" , k, last-1); |
6490 | } |
6491 | /* Proof that the "+1" on the k value above is safe */ |
6492 | pOp = sqlite3VdbeGetOp(v, k - 1); |
6493 | assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur ); |
6494 | assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur ); |
6495 | assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur ); |
6496 | #endif |
6497 | pOp = sqlite3VdbeGetOp(v, k); |
6498 | pLastOp = pOp + (last - k); |
6499 | assert( pOp<=pLastOp ); |
6500 | do{ |
6501 | if( pOp->p1!=pLevel->iTabCur ){ |
6502 | /* no-op */ |
6503 | }else if( pOp->opcode==OP_Column |
6504 | #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC |
6505 | || pOp->opcode==OP_Offset |
6506 | #endif |
6507 | ){ |
6508 | int x = pOp->p2; |
6509 | assert( pIdx->pTable==pTab ); |
6510 | #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC |
6511 | if( pOp->opcode==OP_Offset ){ |
6512 | /* Do not need to translate the column number */ |
6513 | }else |
6514 | #endif |
6515 | if( !HasRowid(pTab) ){ |
6516 | Index *pPk = sqlite3PrimaryKeyIndex(pTab); |
6517 | x = pPk->aiColumn[x]; |
6518 | assert( x>=0 ); |
6519 | }else{ |
6520 | testcase( x!=sqlite3StorageColumnToTable(pTab,x) ); |
6521 | x = sqlite3StorageColumnToTable(pTab,x); |
6522 | } |
6523 | x = sqlite3TableColumnToIndex(pIdx, x); |
6524 | if( x>=0 ){ |
6525 | pOp->p2 = x; |
6526 | pOp->p1 = pLevel->iIdxCur; |
6527 | OpcodeRewriteTrace(db, k, pOp); |
6528 | }else{ |
6529 | /* Unable to translate the table reference into an index |
6530 | ** reference. Verify that this is harmless - that the |
6531 | ** table being referenced really is open. |
6532 | */ |
6533 | #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC |
6534 | assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 |
6535 | || cursorIsOpen(v,pOp->p1,k) |
6536 | || pOp->opcode==OP_Offset |
6537 | ); |
6538 | #else |
6539 | assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 |
6540 | || cursorIsOpen(v,pOp->p1,k) |
6541 | ); |
6542 | #endif |
6543 | } |
6544 | }else if( pOp->opcode==OP_Rowid ){ |
6545 | pOp->p1 = pLevel->iIdxCur; |
6546 | pOp->opcode = OP_IdxRowid; |
6547 | OpcodeRewriteTrace(db, k, pOp); |
6548 | }else if( pOp->opcode==OP_IfNullRow ){ |
6549 | pOp->p1 = pLevel->iIdxCur; |
6550 | OpcodeRewriteTrace(db, k, pOp); |
6551 | } |
6552 | #ifdef SQLITE_DEBUG |
6553 | k++; |
6554 | #endif |
6555 | }while( (++pOp)<pLastOp ); |
6556 | #ifdef SQLITE_DEBUG |
6557 | if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n" ); |
6558 | #endif |
6559 | } |
6560 | } |
6561 | |
6562 | /* The "break" point is here, just past the end of the outer loop. |
6563 | ** Set it. |
6564 | */ |
6565 | sqlite3VdbeResolveLabel(v, pWInfo->iBreak); |
6566 | |
6567 | /* Final cleanup |
6568 | */ |
6569 | pParse->nQueryLoop = pWInfo->savedNQueryLoop; |
6570 | whereInfoFree(db, pWInfo); |
6571 | pParse->withinRJSubrtn -= nRJ; |
6572 | return; |
6573 | } |
6574 | |