1/*
2** 2001 September 15
3**
4** The author disclaims copyright to this source code. In place of
5** a legal notice, here is a blessing:
6**
7** May you do good and not evil.
8** May you find forgiveness for yourself and forgive others.
9** May you share freely, never taking more than you give.
10**
11*************************************************************************
12** This module contains C code that generates VDBE code used to process
13** the WHERE clause of SQL statements. This module is responsible for
14** generating the code that loops through a table looking for applicable
15** rows. Indices are selected and used to speed the search when doing
16** so is applicable. Because this module is responsible for selecting
17** indices, you might also think of this module as the "query optimizer".
18*/
19#include "sqliteInt.h"
20#include "whereInt.h"
21
22/*
23** Extra information appended to the end of sqlite3_index_info but not
24** visible to the xBestIndex function, at least not directly. The
25** sqlite3_vtab_collation() interface knows how to reach it, however.
26**
27** This object is not an API and can be changed from one release to the
28** next. As long as allocateIndexInfo() and sqlite3_vtab_collation()
29** agree on the structure, all will be well.
30*/
31typedef struct HiddenIndexInfo HiddenIndexInfo;
32struct HiddenIndexInfo {
33 WhereClause *pWC; /* The Where clause being analyzed */
34 Parse *pParse; /* The parsing context */
35 int eDistinct; /* Value to return from sqlite3_vtab_distinct() */
36 u32 mIn; /* Mask of terms that are <col> IN (...) */
37 u32 mHandleIn; /* Terms that vtab will handle as <col> IN (...) */
38 sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST
39 ** because extra space is allocated to hold up
40 ** to nTerm such values */
41};
42
43/* Forward declaration of methods */
44static int whereLoopResize(sqlite3*, WhereLoop*, int);
45
46/*
47** Return the estimated number of output rows from a WHERE clause
48*/
49LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
50 return pWInfo->nRowOut;
51}
52
53/*
54** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this
55** WHERE clause returns outputs for DISTINCT processing.
56*/
57int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
58 return pWInfo->eDistinct;
59}
60
61/*
62** Return the number of ORDER BY terms that are satisfied by the
63** WHERE clause. A return of 0 means that the output must be
64** completely sorted. A return equal to the number of ORDER BY
65** terms means that no sorting is needed at all. A return that
66** is positive but less than the number of ORDER BY terms means that
67** block sorting is required.
68*/
69int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
70 return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat;
71}
72
73/*
74** In the ORDER BY LIMIT optimization, if the inner-most loop is known
75** to emit rows in increasing order, and if the last row emitted by the
76** inner-most loop did not fit within the sorter, then we can skip all
77** subsequent rows for the current iteration of the inner loop (because they
78** will not fit in the sorter either) and continue with the second inner
79** loop - the loop immediately outside the inner-most.
80**
81** When a row does not fit in the sorter (because the sorter already
82** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the
83** label returned by this function.
84**
85** If the ORDER BY LIMIT optimization applies, the jump destination should
86** be the continuation for the second-inner-most loop. If the ORDER BY
87** LIMIT optimization does not apply, then the jump destination should
88** be the continuation for the inner-most loop.
89**
90** It is always safe for this routine to return the continuation of the
91** inner-most loop, in the sense that a correct answer will result.
92** Returning the continuation the second inner loop is an optimization
93** that might make the code run a little faster, but should not change
94** the final answer.
95*/
96int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){
97 WhereLevel *pInner;
98 if( !pWInfo->bOrderedInnerLoop ){
99 /* The ORDER BY LIMIT optimization does not apply. Jump to the
100 ** continuation of the inner-most loop. */
101 return pWInfo->iContinue;
102 }
103 pInner = &pWInfo->a[pWInfo->nLevel-1];
104 assert( pInner->addrNxt!=0 );
105 return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt;
106}
107
108/*
109** While generating code for the min/max optimization, after handling
110** the aggregate-step call to min() or max(), check to see if any
111** additional looping is required. If the output order is such that
112** we are certain that the correct answer has already been found, then
113** code an OP_Goto to by pass subsequent processing.
114**
115** Any extra OP_Goto that is coded here is an optimization. The
116** correct answer should be obtained regardless. This OP_Goto just
117** makes the answer appear faster.
118*/
119void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){
120 WhereLevel *pInner;
121 int i;
122 if( !pWInfo->bOrderedInnerLoop ) return;
123 if( pWInfo->nOBSat==0 ) return;
124 for(i=pWInfo->nLevel-1; i>=0; i--){
125 pInner = &pWInfo->a[i];
126 if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){
127 sqlite3VdbeGoto(v, pInner->addrNxt);
128 return;
129 }
130 }
131 sqlite3VdbeGoto(v, pWInfo->iBreak);
132}
133
134/*
135** Return the VDBE address or label to jump to in order to continue
136** immediately with the next row of a WHERE clause.
137*/
138int sqlite3WhereContinueLabel(WhereInfo *pWInfo){
139 assert( pWInfo->iContinue!=0 );
140 return pWInfo->iContinue;
141}
142
143/*
144** Return the VDBE address or label to jump to in order to break
145** out of a WHERE loop.
146*/
147int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
148 return pWInfo->iBreak;
149}
150
151/*
152** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
153** operate directly on the rowids returned by a WHERE clause. Return
154** ONEPASS_SINGLE (1) if the statement can operation directly because only
155** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass
156** optimization can be used on multiple
157**
158** If the ONEPASS optimization is used (if this routine returns true)
159** then also write the indices of open cursors used by ONEPASS
160** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data
161** table and iaCur[1] gets the cursor used by an auxiliary index.
162** Either value may be -1, indicating that cursor is not used.
163** Any cursors returned will have been opened for writing.
164**
165** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is
166** unable to use the ONEPASS optimization.
167*/
168int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
169 memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
170#ifdef WHERETRACE_ENABLED
171 if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){
172 sqlite3DebugPrintf("%s cursors: %d %d\n",
173 pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI",
174 aiCur[0], aiCur[1]);
175 }
176#endif
177 return pWInfo->eOnePass;
178}
179
180/*
181** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move
182** the data cursor to the row selected by the index cursor.
183*/
184int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){
185 return pWInfo->bDeferredSeek;
186}
187
188/*
189** Move the content of pSrc into pDest
190*/
191static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){
192 pDest->n = pSrc->n;
193 memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0]));
194}
195
196/*
197** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet.
198**
199** The new entry might overwrite an existing entry, or it might be
200** appended, or it might be discarded. Do whatever is the right thing
201** so that pSet keeps the N_OR_COST best entries seen so far.
202*/
203static int whereOrInsert(
204 WhereOrSet *pSet, /* The WhereOrSet to be updated */
205 Bitmask prereq, /* Prerequisites of the new entry */
206 LogEst rRun, /* Run-cost of the new entry */
207 LogEst nOut /* Number of outputs for the new entry */
208){
209 u16 i;
210 WhereOrCost *p;
211 for(i=pSet->n, p=pSet->a; i>0; i--, p++){
212 if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){
213 goto whereOrInsert_done;
214 }
215 if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){
216 return 0;
217 }
218 }
219 if( pSet->n<N_OR_COST ){
220 p = &pSet->a[pSet->n++];
221 p->nOut = nOut;
222 }else{
223 p = pSet->a;
224 for(i=1; i<pSet->n; i++){
225 if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i;
226 }
227 if( p->rRun<=rRun ) return 0;
228 }
229whereOrInsert_done:
230 p->prereq = prereq;
231 p->rRun = rRun;
232 if( p->nOut>nOut ) p->nOut = nOut;
233 return 1;
234}
235
236/*
237** Return the bitmask for the given cursor number. Return 0 if
238** iCursor is not in the set.
239*/
240Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){
241 int i;
242 assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 );
243 assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 );
244 assert( iCursor>=-1 );
245 if( pMaskSet->ix[0]==iCursor ){
246 return 1;
247 }
248 for(i=1; i<pMaskSet->n; i++){
249 if( pMaskSet->ix[i]==iCursor ){
250 return MASKBIT(i);
251 }
252 }
253 return 0;
254}
255
256/* Allocate memory that is automatically freed when pWInfo is freed.
257*/
258void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){
259 WhereMemBlock *pBlock;
260 pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock));
261 if( pBlock ){
262 pBlock->pNext = pWInfo->pMemToFree;
263 pBlock->sz = nByte;
264 pWInfo->pMemToFree = pBlock;
265 pBlock++;
266 }
267 return (void*)pBlock;
268}
269void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){
270 void *pNew = sqlite3WhereMalloc(pWInfo, nByte);
271 if( pNew && pOld ){
272 WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld;
273 pOldBlk--;
274 assert( pOldBlk->sz<nByte );
275 memcpy(pNew, pOld, pOldBlk->sz);
276 }
277 return pNew;
278}
279
280/*
281** Create a new mask for cursor iCursor.
282**
283** There is one cursor per table in the FROM clause. The number of
284** tables in the FROM clause is limited by a test early in the
285** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[]
286** array will never overflow.
287*/
288static void createMask(WhereMaskSet *pMaskSet, int iCursor){
289 assert( pMaskSet->n < ArraySize(pMaskSet->ix) );
290 pMaskSet->ix[pMaskSet->n++] = iCursor;
291}
292
293/*
294** If the right-hand branch of the expression is a TK_COLUMN, then return
295** a pointer to the right-hand branch. Otherwise, return NULL.
296*/
297static Expr *whereRightSubexprIsColumn(Expr *p){
298 p = sqlite3ExprSkipCollateAndLikely(p->pRight);
299 if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){
300 return p;
301 }
302 return 0;
303}
304
305/*
306** Advance to the next WhereTerm that matches according to the criteria
307** established when the pScan object was initialized by whereScanInit().
308** Return NULL if there are no more matching WhereTerms.
309*/
310static WhereTerm *whereScanNext(WhereScan *pScan){
311 int iCur; /* The cursor on the LHS of the term */
312 i16 iColumn; /* The column on the LHS of the term. -1 for IPK */
313 Expr *pX; /* An expression being tested */
314 WhereClause *pWC; /* Shorthand for pScan->pWC */
315 WhereTerm *pTerm; /* The term being tested */
316 int k = pScan->k; /* Where to start scanning */
317
318 assert( pScan->iEquiv<=pScan->nEquiv );
319 pWC = pScan->pWC;
320 while(1){
321 iColumn = pScan->aiColumn[pScan->iEquiv-1];
322 iCur = pScan->aiCur[pScan->iEquiv-1];
323 assert( pWC!=0 );
324 assert( iCur>=0 );
325 do{
326 for(pTerm=pWC->a+k; k<pWC->nTerm; k++, pTerm++){
327 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 );
328 if( pTerm->leftCursor==iCur
329 && pTerm->u.x.leftColumn==iColumn
330 && (iColumn!=XN_EXPR
331 || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft,
332 pScan->pIdxExpr,iCur)==0)
333 && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON))
334 ){
335 if( (pTerm->eOperator & WO_EQUIV)!=0
336 && pScan->nEquiv<ArraySize(pScan->aiCur)
337 && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0
338 ){
339 int j;
340 for(j=0; j<pScan->nEquiv; j++){
341 if( pScan->aiCur[j]==pX->iTable
342 && pScan->aiColumn[j]==pX->iColumn ){
343 break;
344 }
345 }
346 if( j==pScan->nEquiv ){
347 pScan->aiCur[j] = pX->iTable;
348 pScan->aiColumn[j] = pX->iColumn;
349 pScan->nEquiv++;
350 }
351 }
352 if( (pTerm->eOperator & pScan->opMask)!=0 ){
353 /* Verify the affinity and collating sequence match */
354 if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
355 CollSeq *pColl;
356 Parse *pParse = pWC->pWInfo->pParse;
357 pX = pTerm->pExpr;
358 if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
359 continue;
360 }
361 assert(pX->pLeft);
362 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
363 if( pColl==0 ) pColl = pParse->db->pDfltColl;
364 if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){
365 continue;
366 }
367 }
368 if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0
369 && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0))
370 && pX->op==TK_COLUMN
371 && pX->iTable==pScan->aiCur[0]
372 && pX->iColumn==pScan->aiColumn[0]
373 ){
374 testcase( pTerm->eOperator & WO_IS );
375 continue;
376 }
377 pScan->pWC = pWC;
378 pScan->k = k+1;
379#ifdef WHERETRACE_ENABLED
380 if( sqlite3WhereTrace & 0x20000 ){
381 int ii;
382 sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d",
383 pTerm, pScan->nEquiv);
384 for(ii=0; ii<pScan->nEquiv; ii++){
385 sqlite3DebugPrintf(" {%d:%d}",
386 pScan->aiCur[ii], pScan->aiColumn[ii]);
387 }
388 sqlite3DebugPrintf("\n");
389 }
390#endif
391 return pTerm;
392 }
393 }
394 }
395 pWC = pWC->pOuter;
396 k = 0;
397 }while( pWC!=0 );
398 if( pScan->iEquiv>=pScan->nEquiv ) break;
399 pWC = pScan->pOrigWC;
400 k = 0;
401 pScan->iEquiv++;
402 }
403 return 0;
404}
405
406/*
407** This is whereScanInit() for the case of an index on an expression.
408** It is factored out into a separate tail-recursion subroutine so that
409** the normal whereScanInit() routine, which is a high-runner, does not
410** need to push registers onto the stack as part of its prologue.
411*/
412static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){
413 pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr);
414 return whereScanNext(pScan);
415}
416
417/*
418** Initialize a WHERE clause scanner object. Return a pointer to the
419** first match. Return NULL if there are no matches.
420**
421** The scanner will be searching the WHERE clause pWC. It will look
422** for terms of the form "X <op> <expr>" where X is column iColumn of table
423** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx
424** must be one of the indexes of table iCur.
425**
426** The <op> must be one of the operators described by opMask.
427**
428** If the search is for X and the WHERE clause contains terms of the
429** form X=Y then this routine might also return terms of the form
430** "Y <op> <expr>". The number of levels of transitivity is limited,
431** but is enough to handle most commonly occurring SQL statements.
432**
433** If X is not the INTEGER PRIMARY KEY then X must be compatible with
434** index pIdx.
435*/
436static WhereTerm *whereScanInit(
437 WhereScan *pScan, /* The WhereScan object being initialized */
438 WhereClause *pWC, /* The WHERE clause to be scanned */
439 int iCur, /* Cursor to scan for */
440 int iColumn, /* Column to scan for */
441 u32 opMask, /* Operator(s) to scan for */
442 Index *pIdx /* Must be compatible with this index */
443){
444 pScan->pOrigWC = pWC;
445 pScan->pWC = pWC;
446 pScan->pIdxExpr = 0;
447 pScan->idxaff = 0;
448 pScan->zCollName = 0;
449 pScan->opMask = opMask;
450 pScan->k = 0;
451 pScan->aiCur[0] = iCur;
452 pScan->nEquiv = 1;
453 pScan->iEquiv = 1;
454 if( pIdx ){
455 int j = iColumn;
456 iColumn = pIdx->aiColumn[j];
457 if( iColumn==pIdx->pTable->iPKey ){
458 iColumn = XN_ROWID;
459 }else if( iColumn>=0 ){
460 pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
461 pScan->zCollName = pIdx->azColl[j];
462 }else if( iColumn==XN_EXPR ){
463 pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
464 pScan->zCollName = pIdx->azColl[j];
465 pScan->aiColumn[0] = XN_EXPR;
466 return whereScanInitIndexExpr(pScan);
467 }
468 }else if( iColumn==XN_EXPR ){
469 return 0;
470 }
471 pScan->aiColumn[0] = iColumn;
472 return whereScanNext(pScan);
473}
474
475/*
476** Search for a term in the WHERE clause that is of the form "X <op> <expr>"
477** where X is a reference to the iColumn of table iCur or of index pIdx
478** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
479** the op parameter. Return a pointer to the term. Return 0 if not found.
480**
481** If pIdx!=0 then it must be one of the indexes of table iCur.
482** Search for terms matching the iColumn-th column of pIdx
483** rather than the iColumn-th column of table iCur.
484**
485** The term returned might by Y=<expr> if there is another constraint in
486** the WHERE clause that specifies that X=Y. Any such constraints will be
487** identified by the WO_EQUIV bit in the pTerm->eOperator field. The
488** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11
489** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10
490** other equivalent values. Hence a search for X will return <expr> if X=A1
491** and A1=A2 and A2=A3 and ... and A9=A10 and A10=<expr>.
492**
493** If there are multiple terms in the WHERE clause of the form "X <op> <expr>"
494** then try for the one with no dependencies on <expr> - in other words where
495** <expr> is a constant expression of some kind. Only return entries of
496** the form "X <op> Y" where Y is a column in another table if no terms of
497** the form "X <op> <const-expr>" exist. If no terms with a constant RHS
498** exist, try to return a term that does not use WO_EQUIV.
499*/
500WhereTerm *sqlite3WhereFindTerm(
501 WhereClause *pWC, /* The WHERE clause to be searched */
502 int iCur, /* Cursor number of LHS */
503 int iColumn, /* Column number of LHS */
504 Bitmask notReady, /* RHS must not overlap with this mask */
505 u32 op, /* Mask of WO_xx values describing operator */
506 Index *pIdx /* Must be compatible with this index, if not NULL */
507){
508 WhereTerm *pResult = 0;
509 WhereTerm *p;
510 WhereScan scan;
511
512 p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx);
513 op &= WO_EQ|WO_IS;
514 while( p ){
515 if( (p->prereqRight & notReady)==0 ){
516 if( p->prereqRight==0 && (p->eOperator&op)!=0 ){
517 testcase( p->eOperator & WO_IS );
518 return p;
519 }
520 if( pResult==0 ) pResult = p;
521 }
522 p = whereScanNext(&scan);
523 }
524 return pResult;
525}
526
527/*
528** This function searches pList for an entry that matches the iCol-th column
529** of index pIdx.
530**
531** If such an expression is found, its index in pList->a[] is returned. If
532** no expression is found, -1 is returned.
533*/
534static int findIndexCol(
535 Parse *pParse, /* Parse context */
536 ExprList *pList, /* Expression list to search */
537 int iBase, /* Cursor for table associated with pIdx */
538 Index *pIdx, /* Index to match column of */
539 int iCol /* Column of index to match */
540){
541 int i;
542 const char *zColl = pIdx->azColl[iCol];
543
544 for(i=0; i<pList->nExpr; i++){
545 Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
546 if( ALWAYS(p!=0)
547 && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
548 && p->iColumn==pIdx->aiColumn[iCol]
549 && p->iTable==iBase
550 ){
551 CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr);
552 if( 0==sqlite3StrICmp(pColl->zName, zColl) ){
553 return i;
554 }
555 }
556 }
557
558 return -1;
559}
560
561/*
562** Return TRUE if the iCol-th column of index pIdx is NOT NULL
563*/
564static int indexColumnNotNull(Index *pIdx, int iCol){
565 int j;
566 assert( pIdx!=0 );
567 assert( iCol>=0 && iCol<pIdx->nColumn );
568 j = pIdx->aiColumn[iCol];
569 if( j>=0 ){
570 return pIdx->pTable->aCol[j].notNull;
571 }else if( j==(-1) ){
572 return 1;
573 }else{
574 assert( j==(-2) );
575 return 0; /* Assume an indexed expression can always yield a NULL */
576
577 }
578}
579
580/*
581** Return true if the DISTINCT expression-list passed as the third argument
582** is redundant.
583**
584** A DISTINCT list is redundant if any subset of the columns in the
585** DISTINCT list are collectively unique and individually non-null.
586*/
587static int isDistinctRedundant(
588 Parse *pParse, /* Parsing context */
589 SrcList *pTabList, /* The FROM clause */
590 WhereClause *pWC, /* The WHERE clause */
591 ExprList *pDistinct /* The result set that needs to be DISTINCT */
592){
593 Table *pTab;
594 Index *pIdx;
595 int i;
596 int iBase;
597
598 /* If there is more than one table or sub-select in the FROM clause of
599 ** this query, then it will not be possible to show that the DISTINCT
600 ** clause is redundant. */
601 if( pTabList->nSrc!=1 ) return 0;
602 iBase = pTabList->a[0].iCursor;
603 pTab = pTabList->a[0].pTab;
604
605 /* If any of the expressions is an IPK column on table iBase, then return
606 ** true. Note: The (p->iTable==iBase) part of this test may be false if the
607 ** current SELECT is a correlated sub-query.
608 */
609 for(i=0; i<pDistinct->nExpr; i++){
610 Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
611 if( NEVER(p==0) ) continue;
612 if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue;
613 if( p->iTable==iBase && p->iColumn<0 ) return 1;
614 }
615
616 /* Loop through all indices on the table, checking each to see if it makes
617 ** the DISTINCT qualifier redundant. It does so if:
618 **
619 ** 1. The index is itself UNIQUE, and
620 **
621 ** 2. All of the columns in the index are either part of the pDistinct
622 ** list, or else the WHERE clause contains a term of the form "col=X",
623 ** where X is a constant value. The collation sequences of the
624 ** comparison and select-list expressions must match those of the index.
625 **
626 ** 3. All of those index columns for which the WHERE clause does not
627 ** contain a "col=X" term are subject to a NOT NULL constraint.
628 */
629 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
630 if( !IsUniqueIndex(pIdx) ) continue;
631 if( pIdx->pPartIdxWhere ) continue;
632 for(i=0; i<pIdx->nKeyCol; i++){
633 if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){
634 if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break;
635 if( indexColumnNotNull(pIdx, i)==0 ) break;
636 }
637 }
638 if( i==pIdx->nKeyCol ){
639 /* This index implies that the DISTINCT qualifier is redundant. */
640 return 1;
641 }
642 }
643
644 return 0;
645}
646
647
648/*
649** Estimate the logarithm of the input value to base 2.
650*/
651static LogEst estLog(LogEst N){
652 return N<=10 ? 0 : sqlite3LogEst(N) - 33;
653}
654
655/*
656** Convert OP_Column opcodes to OP_Copy in previously generated code.
657**
658** This routine runs over generated VDBE code and translates OP_Column
659** opcodes into OP_Copy when the table is being accessed via co-routine
660** instead of via table lookup.
661**
662** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
663** cursor iTabCur are transformed into OP_Sequence opcode for the
664** iAutoidxCur cursor, in order to generate unique rowids for the
665** automatic index being generated.
666*/
667static void translateColumnToCopy(
668 Parse *pParse, /* Parsing context */
669 int iStart, /* Translate from this opcode to the end */
670 int iTabCur, /* OP_Column/OP_Rowid references to this table */
671 int iRegister, /* The first column is in this register */
672 int iAutoidxCur /* If non-zero, cursor of autoindex being generated */
673){
674 Vdbe *v = pParse->pVdbe;
675 VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
676 int iEnd = sqlite3VdbeCurrentAddr(v);
677 if( pParse->db->mallocFailed ) return;
678 for(; iStart<iEnd; iStart++, pOp++){
679 if( pOp->p1!=iTabCur ) continue;
680 if( pOp->opcode==OP_Column ){
681 pOp->opcode = OP_Copy;
682 pOp->p1 = pOp->p2 + iRegister;
683 pOp->p2 = pOp->p3;
684 pOp->p3 = 0;
685 pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */
686 }else if( pOp->opcode==OP_Rowid ){
687 pOp->opcode = OP_Sequence;
688 pOp->p1 = iAutoidxCur;
689#ifdef SQLITE_ALLOW_ROWID_IN_VIEW
690 if( iAutoidxCur==0 ){
691 pOp->opcode = OP_Null;
692 pOp->p3 = 0;
693 }
694#endif
695 }
696 }
697}
698
699/*
700** Two routines for printing the content of an sqlite3_index_info
701** structure. Used for testing and debugging only. If neither
702** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines
703** are no-ops.
704*/
705#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
706static void whereTraceIndexInfoInputs(sqlite3_index_info *p){
707 int i;
708 if( !sqlite3WhereTrace ) return;
709 for(i=0; i<p->nConstraint; i++){
710 sqlite3DebugPrintf(
711 " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n",
712 i,
713 p->aConstraint[i].iColumn,
714 p->aConstraint[i].iTermOffset,
715 p->aConstraint[i].op,
716 p->aConstraint[i].usable,
717 sqlite3_vtab_collation(p,i));
718 }
719 for(i=0; i<p->nOrderBy; i++){
720 sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n",
721 i,
722 p->aOrderBy[i].iColumn,
723 p->aOrderBy[i].desc);
724 }
725}
726static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){
727 int i;
728 if( !sqlite3WhereTrace ) return;
729 for(i=0; i<p->nConstraint; i++){
730 sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n",
731 i,
732 p->aConstraintUsage[i].argvIndex,
733 p->aConstraintUsage[i].omit);
734 }
735 sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum);
736 sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr);
737 sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed);
738 sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost);
739 sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows);
740}
741#else
742#define whereTraceIndexInfoInputs(A)
743#define whereTraceIndexInfoOutputs(A)
744#endif
745
746/*
747** We know that pSrc is an operand of an outer join. Return true if
748** pTerm is a constraint that is compatible with that join.
749**
750** pTerm must be EP_OuterON if pSrc is the right operand of an
751** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc
752** is the left operand of a RIGHT join.
753**
754** See https://sqlite.org/forum/forumpost/206d99a16dd9212f
755** for an example of a WHERE clause constraints that may not be used on
756** the right table of a RIGHT JOIN because the constraint implies a
757** not-NULL condition on the left table of the RIGHT JOIN.
758*/
759static int constraintCompatibleWithOuterJoin(
760 const WhereTerm *pTerm, /* WHERE clause term to check */
761 const SrcItem *pSrc /* Table we are trying to access */
762){
763 assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */
764 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT );
765 testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ );
766 testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) )
767 testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) );
768 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON)
769 || pTerm->pExpr->w.iJoin != pSrc->iCursor
770 ){
771 return 0;
772 }
773 if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0
774 && ExprHasProperty(pTerm->pExpr, EP_InnerON)
775 ){
776 return 0;
777 }
778 return 1;
779}
780
781
782
783#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
784/*
785** Return TRUE if the WHERE clause term pTerm is of a form where it
786** could be used with an index to access pSrc, assuming an appropriate
787** index existed.
788*/
789static int termCanDriveIndex(
790 const WhereTerm *pTerm, /* WHERE clause term to check */
791 const SrcItem *pSrc, /* Table we are trying to access */
792 const Bitmask notReady /* Tables in outer loops of the join */
793){
794 char aff;
795 if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
796 if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
797 assert( (pSrc->fg.jointype & JT_RIGHT)==0 );
798 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
799 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
800 ){
801 return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */
802 }
803 if( (pTerm->prereqRight & notReady)!=0 ) return 0;
804 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
805 if( pTerm->u.x.leftColumn<0 ) return 0;
806 aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity;
807 if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
808 testcase( pTerm->pExpr->op==TK_IS );
809 return 1;
810}
811#endif
812
813
814#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
815/*
816** Generate code to construct the Index object for an automatic index
817** and to set up the WhereLevel object pLevel so that the code generator
818** makes use of the automatic index.
819*/
820static SQLITE_NOINLINE void constructAutomaticIndex(
821 Parse *pParse, /* The parsing context */
822 const WhereClause *pWC, /* The WHERE clause */
823 const SrcItem *pSrc, /* The FROM clause term to get the next index */
824 const Bitmask notReady, /* Mask of cursors that are not available */
825 WhereLevel *pLevel /* Write new index here */
826){
827 int nKeyCol; /* Number of columns in the constructed index */
828 WhereTerm *pTerm; /* A single term of the WHERE clause */
829 WhereTerm *pWCEnd; /* End of pWC->a[] */
830 Index *pIdx; /* Object describing the transient index */
831 Vdbe *v; /* Prepared statement under construction */
832 int addrInit; /* Address of the initialization bypass jump */
833 Table *pTable; /* The table being indexed */
834 int addrTop; /* Top of the index fill loop */
835 int regRecord; /* Register holding an index record */
836 int n; /* Column counter */
837 int i; /* Loop counter */
838 int mxBitCol; /* Maximum column in pSrc->colUsed */
839 CollSeq *pColl; /* Collating sequence to on a column */
840 WhereLoop *pLoop; /* The Loop object */
841 char *zNotUsed; /* Extra space on the end of pIdx */
842 Bitmask idxCols; /* Bitmap of columns used for indexing */
843 Bitmask extraCols; /* Bitmap of additional columns */
844 u8 sentWarning = 0; /* True if a warnning has been issued */
845 Expr *pPartial = 0; /* Partial Index Expression */
846 int iContinue = 0; /* Jump here to skip excluded rows */
847 SrcItem *pTabItem; /* FROM clause term being indexed */
848 int addrCounter = 0; /* Address where integer counter is initialized */
849 int regBase; /* Array of registers where record is assembled */
850
851 /* Generate code to skip over the creation and initialization of the
852 ** transient index on 2nd and subsequent iterations of the loop. */
853 v = pParse->pVdbe;
854 assert( v!=0 );
855 addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
856
857 /* Count the number of columns that will be added to the index
858 ** and used to match WHERE clause constraints */
859 nKeyCol = 0;
860 pTable = pSrc->pTab;
861 pWCEnd = &pWC->a[pWC->nTerm];
862 pLoop = pLevel->pWLoop;
863 idxCols = 0;
864 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
865 Expr *pExpr = pTerm->pExpr;
866 /* Make the automatic index a partial index if there are terms in the
867 ** WHERE clause (or the ON clause of a LEFT join) that constrain which
868 ** rows of the target table (pSrc) that can be used. */
869 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
870 && sqlite3ExprIsTableConstraint(pExpr, pSrc)
871 ){
872 pPartial = sqlite3ExprAnd(pParse, pPartial,
873 sqlite3ExprDup(pParse->db, pExpr, 0));
874 }
875 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
876 int iCol;
877 Bitmask cMask;
878 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
879 iCol = pTerm->u.x.leftColumn;
880 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
881 testcase( iCol==BMS );
882 testcase( iCol==BMS-1 );
883 if( !sentWarning ){
884 sqlite3_log(SQLITE_WARNING_AUTOINDEX,
885 "automatic index on %s(%s)", pTable->zName,
886 pTable->aCol[iCol].zCnName);
887 sentWarning = 1;
888 }
889 if( (idxCols & cMask)==0 ){
890 if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){
891 goto end_auto_index_create;
892 }
893 pLoop->aLTerm[nKeyCol++] = pTerm;
894 idxCols |= cMask;
895 }
896 }
897 }
898 assert( nKeyCol>0 || pParse->db->mallocFailed );
899 pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol;
900 pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED
901 | WHERE_AUTO_INDEX;
902
903 /* Count the number of additional columns needed to create a
904 ** covering index. A "covering index" is an index that contains all
905 ** columns that are needed by the query. With a covering index, the
906 ** original table never needs to be accessed. Automatic indices must
907 ** be a covering index because the index will not be updated if the
908 ** original table changes and the index and table cannot both be used
909 ** if they go out of sync.
910 */
911 extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
912 mxBitCol = MIN(BMS-1,pTable->nCol);
913 testcase( pTable->nCol==BMS-1 );
914 testcase( pTable->nCol==BMS-2 );
915 for(i=0; i<mxBitCol; i++){
916 if( extraCols & MASKBIT(i) ) nKeyCol++;
917 }
918 if( pSrc->colUsed & MASKBIT(BMS-1) ){
919 nKeyCol += pTable->nCol - BMS + 1;
920 }
921
922 /* Construct the Index object to describe this index */
923 pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed);
924 if( pIdx==0 ) goto end_auto_index_create;
925 pLoop->u.btree.pIndex = pIdx;
926 pIdx->zName = "auto-index";
927 pIdx->pTable = pTable;
928 n = 0;
929 idxCols = 0;
930 for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
931 if( termCanDriveIndex(pTerm, pSrc, notReady) ){
932 int iCol;
933 Bitmask cMask;
934 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
935 iCol = pTerm->u.x.leftColumn;
936 cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol);
937 testcase( iCol==BMS-1 );
938 testcase( iCol==BMS );
939 if( (idxCols & cMask)==0 ){
940 Expr *pX = pTerm->pExpr;
941 idxCols |= cMask;
942 pIdx->aiColumn[n] = pTerm->u.x.leftColumn;
943 pColl = sqlite3ExprCompareCollSeq(pParse, pX);
944 assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */
945 pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
946 n++;
947 }
948 }
949 }
950 assert( (u32)n==pLoop->u.btree.nEq );
951
952 /* Add additional columns needed to make the automatic index into
953 ** a covering index */
954 for(i=0; i<mxBitCol; i++){
955 if( extraCols & MASKBIT(i) ){
956 pIdx->aiColumn[n] = i;
957 pIdx->azColl[n] = sqlite3StrBINARY;
958 n++;
959 }
960 }
961 if( pSrc->colUsed & MASKBIT(BMS-1) ){
962 for(i=BMS-1; i<pTable->nCol; i++){
963 pIdx->aiColumn[n] = i;
964 pIdx->azColl[n] = sqlite3StrBINARY;
965 n++;
966 }
967 }
968 assert( n==nKeyCol );
969 pIdx->aiColumn[n] = XN_ROWID;
970 pIdx->azColl[n] = sqlite3StrBINARY;
971
972 /* Create the automatic index */
973 assert( pLevel->iIdxCur>=0 );
974 pLevel->iIdxCur = pParse->nTab++;
975 sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
976 sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
977 VdbeComment((v, "for %s", pTable->zName));
978 if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){
979 pLevel->regFilter = ++pParse->nMem;
980 sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
981 }
982
983 /* Fill the automatic index with content */
984 pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom];
985 if( pTabItem->fg.viaCoroutine ){
986 int regYield = pTabItem->regReturn;
987 addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
988 sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub);
989 addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield);
990 VdbeCoverage(v);
991 VdbeComment((v, "next row of %s", pTabItem->pTab->zName));
992 }else{
993 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
994 }
995 if( pPartial ){
996 iContinue = sqlite3VdbeMakeLabel(pParse);
997 sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL);
998 pLoop->wsFlags |= WHERE_PARTIALIDX;
999 }
1000 regRecord = sqlite3GetTempReg(pParse);
1001 regBase = sqlite3GenerateIndexKey(
1002 pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0
1003 );
1004 if( pLevel->regFilter ){
1005 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
1006 regBase, pLoop->u.btree.nEq);
1007 }
1008 sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
1009 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
1010 if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
1011 if( pTabItem->fg.viaCoroutine ){
1012 sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
1013 testcase( pParse->db->mallocFailed );
1014 assert( pLevel->iIdxCur>0 );
1015 translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
1016 pTabItem->regResult, pLevel->iIdxCur);
1017 sqlite3VdbeGoto(v, addrTop);
1018 pTabItem->fg.viaCoroutine = 0;
1019 }else{
1020 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
1021 sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
1022 }
1023 sqlite3VdbeJumpHere(v, addrTop);
1024 sqlite3ReleaseTempReg(pParse, regRecord);
1025
1026 /* Jump here when skipping the initialization */
1027 sqlite3VdbeJumpHere(v, addrInit);
1028
1029end_auto_index_create:
1030 sqlite3ExprDelete(pParse->db, pPartial);
1031}
1032#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
1033
1034/*
1035** Generate bytecode that will initialize a Bloom filter that is appropriate
1036** for pLevel.
1037**
1038** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER
1039** flag set, initialize a Bloomfilter for them as well. Except don't do
1040** this recursive initialization if the SQLITE_BloomPulldown optimization has
1041** been turned off.
1042**
1043** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared
1044** from the loop, but the regFilter value is set to a register that implements
1045** the Bloom filter. When regFilter is positive, the
1046** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter
1047** and skip the subsequence B-Tree seek if the Bloom filter indicates that
1048** no matching rows exist.
1049**
1050** This routine may only be called if it has previously been determined that
1051** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit
1052** is set.
1053*/
1054static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
1055 WhereInfo *pWInfo, /* The WHERE clause */
1056 int iLevel, /* Index in pWInfo->a[] that is pLevel */
1057 WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */
1058 Bitmask notReady /* Loops that are not ready */
1059){
1060 int addrOnce; /* Address of opening OP_Once */
1061 int addrTop; /* Address of OP_Rewind */
1062 int addrCont; /* Jump here to skip a row */
1063 const WhereTerm *pTerm; /* For looping over WHERE clause terms */
1064 const WhereTerm *pWCEnd; /* Last WHERE clause term */
1065 Parse *pParse = pWInfo->pParse; /* Parsing context */
1066 Vdbe *v = pParse->pVdbe; /* VDBE under construction */
1067 WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */
1068 int iCur; /* Cursor for table getting the filter */
1069
1070 assert( pLoop!=0 );
1071 assert( v!=0 );
1072 assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
1073
1074 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
1075 do{
1076 const SrcItem *pItem;
1077 const Table *pTab;
1078 u64 sz;
1079 sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
1080 addrCont = sqlite3VdbeMakeLabel(pParse);
1081 iCur = pLevel->iTabCur;
1082 pLevel->regFilter = ++pParse->nMem;
1083
1084 /* The Bloom filter is a Blob held in a register. Initialize it
1085 ** to zero-filled blob of at least 80K bits, but maybe more if the
1086 ** estimated size of the table is larger. We could actually
1087 ** measure the size of the table at run-time using OP_Count with
1088 ** P3==1 and use that value to initialize the blob. But that makes
1089 ** testing complicated. By basing the blob size on the value in the
1090 ** sqlite_stat1 table, testing is much easier.
1091 */
1092 pItem = &pWInfo->pTabList->a[pLevel->iFrom];
1093 assert( pItem!=0 );
1094 pTab = pItem->pTab;
1095 assert( pTab!=0 );
1096 sz = sqlite3LogEstToInt(pTab->nRowLogEst);
1097 if( sz<10000 ){
1098 sz = 10000;
1099 }else if( sz>10000000 ){
1100 sz = 10000000;
1101 }
1102 sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter);
1103
1104 addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
1105 pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
1106 for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
1107 Expr *pExpr = pTerm->pExpr;
1108 if( (pTerm->wtFlags & TERM_VIRTUAL)==0
1109 && sqlite3ExprIsTableConstraint(pExpr, pItem)
1110 ){
1111 sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
1112 }
1113 }
1114 if( pLoop->wsFlags & WHERE_IPK ){
1115 int r1 = sqlite3GetTempReg(pParse);
1116 sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
1117 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1);
1118 sqlite3ReleaseTempReg(pParse, r1);
1119 }else{
1120 Index *pIdx = pLoop->u.btree.pIndex;
1121 int n = pLoop->u.btree.nEq;
1122 int r1 = sqlite3GetTempRange(pParse, n);
1123 int jj;
1124 for(jj=0; jj<n; jj++){
1125 int iCol = pIdx->aiColumn[jj];
1126 assert( pIdx->pTable==pItem->pTab );
1127 sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj);
1128 }
1129 sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
1130 sqlite3ReleaseTempRange(pParse, r1, n);
1131 }
1132 sqlite3VdbeResolveLabel(v, addrCont);
1133 sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
1134 VdbeCoverage(v);
1135 sqlite3VdbeJumpHere(v, addrTop);
1136 pLoop->wsFlags &= ~WHERE_BLOOMFILTER;
1137 if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break;
1138 while( ++iLevel < pWInfo->nLevel ){
1139 const SrcItem *pTabItem;
1140 pLevel = &pWInfo->a[iLevel];
1141 pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
1142 if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue;
1143 pLoop = pLevel->pWLoop;
1144 if( NEVER(pLoop==0) ) continue;
1145 if( pLoop->prereq & notReady ) continue;
1146 if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN))
1147 ==WHERE_BLOOMFILTER
1148 ){
1149 /* This is a candidate for bloom-filter pull-down (early evaluation).
1150 ** The test that WHERE_COLUMN_IN is omitted is important, as we are
1151 ** not able to do early evaluation of bloom filters that make use of
1152 ** the IN operator */
1153 break;
1154 }
1155 }
1156 }while( iLevel < pWInfo->nLevel );
1157 sqlite3VdbeJumpHere(v, addrOnce);
1158}
1159
1160
1161#ifndef SQLITE_OMIT_VIRTUALTABLE
1162/*
1163** Allocate and populate an sqlite3_index_info structure. It is the
1164** responsibility of the caller to eventually release the structure
1165** by passing the pointer returned by this function to freeIndexInfo().
1166*/
1167static sqlite3_index_info *allocateIndexInfo(
1168 WhereInfo *pWInfo, /* The WHERE clause */
1169 WhereClause *pWC, /* The WHERE clause being analyzed */
1170 Bitmask mUnusable, /* Ignore terms with these prereqs */
1171 SrcItem *pSrc, /* The FROM clause term that is the vtab */
1172 u16 *pmNoOmit /* Mask of terms not to omit */
1173){
1174 int i, j;
1175 int nTerm;
1176 Parse *pParse = pWInfo->pParse;
1177 struct sqlite3_index_constraint *pIdxCons;
1178 struct sqlite3_index_orderby *pIdxOrderBy;
1179 struct sqlite3_index_constraint_usage *pUsage;
1180 struct HiddenIndexInfo *pHidden;
1181 WhereTerm *pTerm;
1182 int nOrderBy;
1183 sqlite3_index_info *pIdxInfo;
1184 u16 mNoOmit = 0;
1185 const Table *pTab;
1186 int eDistinct = 0;
1187 ExprList *pOrderBy = pWInfo->pOrderBy;
1188
1189 assert( pSrc!=0 );
1190 pTab = pSrc->pTab;
1191 assert( pTab!=0 );
1192 assert( IsVirtual(pTab) );
1193
1194 /* Find all WHERE clause constraints referring to this virtual table.
1195 ** Mark each term with the TERM_OK flag. Set nTerm to the number of
1196 ** terms found.
1197 */
1198 for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
1199 pTerm->wtFlags &= ~TERM_OK;
1200 if( pTerm->leftCursor != pSrc->iCursor ) continue;
1201 if( pTerm->prereqRight & mUnusable ) continue;
1202 assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
1203 testcase( pTerm->eOperator & WO_IN );
1204 testcase( pTerm->eOperator & WO_ISNULL );
1205 testcase( pTerm->eOperator & WO_IS );
1206 testcase( pTerm->eOperator & WO_ALL );
1207 if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
1208 if( pTerm->wtFlags & TERM_VNULL ) continue;
1209
1210 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
1211 assert( pTerm->u.x.leftColumn>=XN_ROWID );
1212 assert( pTerm->u.x.leftColumn<pTab->nCol );
1213 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
1214 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
1215 ){
1216 continue;
1217 }
1218 nTerm++;
1219 pTerm->wtFlags |= TERM_OK;
1220 }
1221
1222 /* If the ORDER BY clause contains only columns in the current
1223 ** virtual table then allocate space for the aOrderBy part of
1224 ** the sqlite3_index_info structure.
1225 */
1226 nOrderBy = 0;
1227 if( pOrderBy ){
1228 int n = pOrderBy->nExpr;
1229 for(i=0; i<n; i++){
1230 Expr *pExpr = pOrderBy->a[i].pExpr;
1231 Expr *pE2;
1232
1233 /* Skip over constant terms in the ORDER BY clause */
1234 if( sqlite3ExprIsConstant(pExpr) ){
1235 continue;
1236 }
1237
1238 /* Virtual tables are unable to deal with NULLS FIRST */
1239 if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break;
1240
1241 /* First case - a direct column references without a COLLATE operator */
1242 if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){
1243 assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumn<pTab->nCol );
1244 continue;
1245 }
1246
1247 /* 2nd case - a column reference with a COLLATE operator. Only match
1248 ** of the COLLATE operator matches the collation of the column. */
1249 if( pExpr->op==TK_COLLATE
1250 && (pE2 = pExpr->pLeft)->op==TK_COLUMN
1251 && pE2->iTable==pSrc->iCursor
1252 ){
1253 const char *zColl; /* The collating sequence name */
1254 assert( !ExprHasProperty(pExpr, EP_IntValue) );
1255 assert( pExpr->u.zToken!=0 );
1256 assert( pE2->iColumn>=XN_ROWID && pE2->iColumn<pTab->nCol );
1257 pExpr->iColumn = pE2->iColumn;
1258 if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */
1259 zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]);
1260 if( zColl==0 ) zColl = sqlite3StrBINARY;
1261 if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue;
1262 }
1263
1264 /* No matches cause a break out of the loop */
1265 break;
1266 }
1267 if( i==n ){
1268 nOrderBy = n;
1269 if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){
1270 eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0);
1271 }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){
1272 eDistinct = 1;
1273 }
1274 }
1275 }
1276
1277 /* Allocate the sqlite3_index_info structure
1278 */
1279 pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
1280 + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
1281 + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden)
1282 + sizeof(sqlite3_value*)*nTerm );
1283 if( pIdxInfo==0 ){
1284 sqlite3ErrorMsg(pParse, "out of memory");
1285 return 0;
1286 }
1287 pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1];
1288 pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm];
1289 pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm];
1290 pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy];
1291 pIdxInfo->aConstraint = pIdxCons;
1292 pIdxInfo->aOrderBy = pIdxOrderBy;
1293 pIdxInfo->aConstraintUsage = pUsage;
1294 pHidden->pWC = pWC;
1295 pHidden->pParse = pParse;
1296 pHidden->eDistinct = eDistinct;
1297 pHidden->mIn = 0;
1298 for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
1299 u16 op;
1300 if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
1301 pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
1302 pIdxCons[j].iTermOffset = i;
1303 op = pTerm->eOperator & WO_ALL;
1304 if( op==WO_IN ){
1305 if( (pTerm->wtFlags & TERM_SLICE)==0 ){
1306 pHidden->mIn |= SMASKBIT32(j);
1307 }
1308 op = WO_EQ;
1309 }
1310 if( op==WO_AUX ){
1311 pIdxCons[j].op = pTerm->eMatchOp;
1312 }else if( op & (WO_ISNULL|WO_IS) ){
1313 if( op==WO_ISNULL ){
1314 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
1315 }else{
1316 pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
1317 }
1318 }else{
1319 pIdxCons[j].op = (u8)op;
1320 /* The direct assignment in the previous line is possible only because
1321 ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The
1322 ** following asserts verify this fact. */
1323 assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
1324 assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
1325 assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
1326 assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
1327 assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
1328 assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
1329
1330 if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
1331 && sqlite3ExprIsVector(pTerm->pExpr->pRight)
1332 ){
1333 testcase( j!=i );
1334 if( j<16 ) mNoOmit |= (1 << j);
1335 if( op==WO_LT ) pIdxCons[j].op = WO_LE;
1336 if( op==WO_GT ) pIdxCons[j].op = WO_GE;
1337 }
1338 }
1339
1340 j++;
1341 }
1342 assert( j==nTerm );
1343 pIdxInfo->nConstraint = j;
1344 for(i=j=0; i<nOrderBy; i++){
1345 Expr *pExpr = pOrderBy->a[i].pExpr;
1346 if( sqlite3ExprIsConstant(pExpr) ) continue;
1347 assert( pExpr->op==TK_COLUMN
1348 || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN
1349 && pExpr->iColumn==pExpr->pLeft->iColumn) );
1350 pIdxOrderBy[j].iColumn = pExpr->iColumn;
1351 pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC;
1352 j++;
1353 }
1354 pIdxInfo->nOrderBy = j;
1355
1356 *pmNoOmit = mNoOmit;
1357 return pIdxInfo;
1358}
1359
1360/*
1361** Free an sqlite3_index_info structure allocated by allocateIndexInfo()
1362** and possibly modified by xBestIndex methods.
1363*/
1364static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
1365 HiddenIndexInfo *pHidden;
1366 int i;
1367 assert( pIdxInfo!=0 );
1368 pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
1369 assert( pHidden->pParse!=0 );
1370 assert( pHidden->pParse->db==db );
1371 for(i=0; i<pIdxInfo->nConstraint; i++){
1372 sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */
1373 pHidden->aRhs[i] = 0;
1374 }
1375 sqlite3DbFree(db, pIdxInfo);
1376}
1377
1378/*
1379** The table object reference passed as the second argument to this function
1380** must represent a virtual table. This function invokes the xBestIndex()
1381** method of the virtual table with the sqlite3_index_info object that
1382** comes in as the 3rd argument to this function.
1383**
1384** If an error occurs, pParse is populated with an error message and an
1385** appropriate error code is returned. A return of SQLITE_CONSTRAINT from
1386** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that
1387** the current configuration of "unusable" flags in sqlite3_index_info can
1388** not result in a valid plan.
1389**
1390** Whether or not an error is returned, it is the responsibility of the
1391** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
1392** that this is required.
1393*/
1394static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
1395 sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
1396 int rc;
1397
1398 whereTraceIndexInfoInputs(p);
1399 pParse->db->nSchemaLock++;
1400 rc = pVtab->pModule->xBestIndex(pVtab, p);
1401 pParse->db->nSchemaLock--;
1402 whereTraceIndexInfoOutputs(p);
1403
1404 if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
1405 if( rc==SQLITE_NOMEM ){
1406 sqlite3OomFault(pParse->db);
1407 }else if( !pVtab->zErrMsg ){
1408 sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
1409 }else{
1410 sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
1411 }
1412 }
1413 sqlite3_free(pVtab->zErrMsg);
1414 pVtab->zErrMsg = 0;
1415 return rc;
1416}
1417#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
1418
1419#ifdef SQLITE_ENABLE_STAT4
1420/*
1421** Estimate the location of a particular key among all keys in an
1422** index. Store the results in aStat as follows:
1423**
1424** aStat[0] Est. number of rows less than pRec
1425** aStat[1] Est. number of rows equal to pRec
1426**
1427** Return the index of the sample that is the smallest sample that
1428** is greater than or equal to pRec. Note that this index is not an index
1429** into the aSample[] array - it is an index into a virtual set of samples
1430** based on the contents of aSample[] and the number of fields in record
1431** pRec.
1432*/
1433static int whereKeyStats(
1434 Parse *pParse, /* Database connection */
1435 Index *pIdx, /* Index to consider domain of */
1436 UnpackedRecord *pRec, /* Vector of values to consider */
1437 int roundUp, /* Round up if true. Round down if false */
1438 tRowcnt *aStat /* OUT: stats written here */
1439){
1440 IndexSample *aSample = pIdx->aSample;
1441 int iCol; /* Index of required stats in anEq[] etc. */
1442 int i; /* Index of first sample >= pRec */
1443 int iSample; /* Smallest sample larger than or equal to pRec */
1444 int iMin = 0; /* Smallest sample not yet tested */
1445 int iTest; /* Next sample to test */
1446 int res; /* Result of comparison operation */
1447 int nField; /* Number of fields in pRec */
1448 tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */
1449
1450#ifndef SQLITE_DEBUG
1451 UNUSED_PARAMETER( pParse );
1452#endif
1453 assert( pRec!=0 );
1454 assert( pIdx->nSample>0 );
1455 assert( pRec->nField>0 );
1456
1457 /* Do a binary search to find the first sample greater than or equal
1458 ** to pRec. If pRec contains a single field, the set of samples to search
1459 ** is simply the aSample[] array. If the samples in aSample[] contain more
1460 ** than one fields, all fields following the first are ignored.
1461 **
1462 ** If pRec contains N fields, where N is more than one, then as well as the
1463 ** samples in aSample[] (truncated to N fields), the search also has to
1464 ** consider prefixes of those samples. For example, if the set of samples
1465 ** in aSample is:
1466 **
1467 ** aSample[0] = (a, 5)
1468 ** aSample[1] = (a, 10)
1469 ** aSample[2] = (b, 5)
1470 ** aSample[3] = (c, 100)
1471 ** aSample[4] = (c, 105)
1472 **
1473 ** Then the search space should ideally be the samples above and the
1474 ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
1475 ** the code actually searches this set:
1476 **
1477 ** 0: (a)
1478 ** 1: (a, 5)
1479 ** 2: (a, 10)
1480 ** 3: (a, 10)
1481 ** 4: (b)
1482 ** 5: (b, 5)
1483 ** 6: (c)
1484 ** 7: (c, 100)
1485 ** 8: (c, 105)
1486 ** 9: (c, 105)
1487 **
1488 ** For each sample in the aSample[] array, N samples are present in the
1489 ** effective sample array. In the above, samples 0 and 1 are based on
1490 ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
1491 **
1492 ** Often, sample i of each block of N effective samples has (i+1) fields.
1493 ** Except, each sample may be extended to ensure that it is greater than or
1494 ** equal to the previous sample in the array. For example, in the above,
1495 ** sample 2 is the first sample of a block of N samples, so at first it
1496 ** appears that it should be 1 field in size. However, that would make it
1497 ** smaller than sample 1, so the binary search would not work. As a result,
1498 ** it is extended to two fields. The duplicates that this creates do not
1499 ** cause any problems.
1500 */
1501 nField = MIN(pRec->nField, pIdx->nSample);
1502 iCol = 0;
1503 iSample = pIdx->nSample * nField;
1504 do{
1505 int iSamp; /* Index in aSample[] of test sample */
1506 int n; /* Number of fields in test sample */
1507
1508 iTest = (iMin+iSample)/2;
1509 iSamp = iTest / nField;
1510 if( iSamp>0 ){
1511 /* The proposed effective sample is a prefix of sample aSample[iSamp].
1512 ** Specifically, the shortest prefix of at least (1 + iTest%nField)
1513 ** fields that is greater than the previous effective sample. */
1514 for(n=(iTest % nField) + 1; n<nField; n++){
1515 if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break;
1516 }
1517 }else{
1518 n = iTest + 1;
1519 }
1520
1521 pRec->nField = n;
1522 res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec);
1523 if( res<0 ){
1524 iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1];
1525 iMin = iTest+1;
1526 }else if( res==0 && n<nField ){
1527 iLower = aSample[iSamp].anLt[n-1];
1528 iMin = iTest+1;
1529 res = -1;
1530 }else{
1531 iSample = iTest;
1532 iCol = n-1;
1533 }
1534 }while( res && iMin<iSample );
1535 i = iSample / nField;
1536
1537#ifdef SQLITE_DEBUG
1538 /* The following assert statements check that the binary search code
1539 ** above found the right answer. This block serves no purpose other
1540 ** than to invoke the asserts. */
1541 if( pParse->db->mallocFailed==0 ){
1542 if( res==0 ){
1543 /* If (res==0) is true, then pRec must be equal to sample i. */
1544 assert( i<pIdx->nSample );
1545 assert( iCol==nField-1 );
1546 pRec->nField = nField;
1547 assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)
1548 || pParse->db->mallocFailed
1549 );
1550 }else{
1551 /* Unless i==pIdx->nSample, indicating that pRec is larger than
1552 ** all samples in the aSample[] array, pRec must be smaller than the
1553 ** (iCol+1) field prefix of sample i. */
1554 assert( i<=pIdx->nSample && i>=0 );
1555 pRec->nField = iCol+1;
1556 assert( i==pIdx->nSample
1557 || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0
1558 || pParse->db->mallocFailed );
1559
1560 /* if i==0 and iCol==0, then record pRec is smaller than all samples
1561 ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must
1562 ** be greater than or equal to the (iCol) field prefix of sample i.
1563 ** If (i>0), then pRec must also be greater than sample (i-1). */
1564 if( iCol>0 ){
1565 pRec->nField = iCol;
1566 assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0
1567 || pParse->db->mallocFailed );
1568 }
1569 if( i>0 ){
1570 pRec->nField = nField;
1571 assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0
1572 || pParse->db->mallocFailed );
1573 }
1574 }
1575 }
1576#endif /* ifdef SQLITE_DEBUG */
1577
1578 if( res==0 ){
1579 /* Record pRec is equal to sample i */
1580 assert( iCol==nField-1 );
1581 aStat[0] = aSample[i].anLt[iCol];
1582 aStat[1] = aSample[i].anEq[iCol];
1583 }else{
1584 /* At this point, the (iCol+1) field prefix of aSample[i] is the first
1585 ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
1586 ** is larger than all samples in the array. */
1587 tRowcnt iUpper, iGap;
1588 if( i>=pIdx->nSample ){
1589 iUpper = pIdx->nRowEst0;
1590 }else{
1591 iUpper = aSample[i].anLt[iCol];
1592 }
1593
1594 if( iLower>=iUpper ){
1595 iGap = 0;
1596 }else{
1597 iGap = iUpper - iLower;
1598 }
1599 if( roundUp ){
1600 iGap = (iGap*2)/3;
1601 }else{
1602 iGap = iGap/3;
1603 }
1604 aStat[0] = iLower + iGap;
1605 aStat[1] = pIdx->aAvgEq[nField-1];
1606 }
1607
1608 /* Restore the pRec->nField value before returning. */
1609 pRec->nField = nField;
1610 return i;
1611}
1612#endif /* SQLITE_ENABLE_STAT4 */
1613
1614/*
1615** If it is not NULL, pTerm is a term that provides an upper or lower
1616** bound on a range scan. Without considering pTerm, it is estimated
1617** that the scan will visit nNew rows. This function returns the number
1618** estimated to be visited after taking pTerm into account.
1619**
1620** If the user explicitly specified a likelihood() value for this term,
1621** then the return value is the likelihood multiplied by the number of
1622** input rows. Otherwise, this function assumes that an "IS NOT NULL" term
1623** has a likelihood of 0.50, and any other term a likelihood of 0.25.
1624*/
1625static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
1626 LogEst nRet = nNew;
1627 if( pTerm ){
1628 if( pTerm->truthProb<=0 ){
1629 nRet += pTerm->truthProb;
1630 }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){
1631 nRet -= 20; assert( 20==sqlite3LogEst(4) );
1632 }
1633 }
1634 return nRet;
1635}
1636
1637
1638#ifdef SQLITE_ENABLE_STAT4
1639/*
1640** Return the affinity for a single column of an index.
1641*/
1642char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
1643 assert( iCol>=0 && iCol<pIdx->nColumn );
1644 if( !pIdx->zColAff ){
1645 if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
1646 }
1647 assert( pIdx->zColAff[iCol]!=0 );
1648 return pIdx->zColAff[iCol];
1649}
1650#endif
1651
1652
1653#ifdef SQLITE_ENABLE_STAT4
1654/*
1655** This function is called to estimate the number of rows visited by a
1656** range-scan on a skip-scan index. For example:
1657**
1658** CREATE INDEX i1 ON t1(a, b, c);
1659** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
1660**
1661** Value pLoop->nOut is currently set to the estimated number of rows
1662** visited for scanning (a=? AND b=?). This function reduces that estimate
1663** by some factor to account for the (c BETWEEN ? AND ?) expression based
1664** on the stat4 data for the index. this scan will be peformed multiple
1665** times (once for each (a,b) combination that matches a=?) is dealt with
1666** by the caller.
1667**
1668** It does this by scanning through all stat4 samples, comparing values
1669** extracted from pLower and pUpper with the corresponding column in each
1670** sample. If L and U are the number of samples found to be less than or
1671** equal to the values extracted from pLower and pUpper respectively, and
1672** N is the total number of samples, the pLoop->nOut value is adjusted
1673** as follows:
1674**
1675** nOut = nOut * ( min(U - L, 1) / N )
1676**
1677** If pLower is NULL, or a value cannot be extracted from the term, L is
1678** set to zero. If pUpper is NULL, or a value cannot be extracted from it,
1679** U is set to N.
1680**
1681** Normally, this function sets *pbDone to 1 before returning. However,
1682** if no value can be extracted from either pLower or pUpper (and so the
1683** estimate of the number of rows delivered remains unchanged), *pbDone
1684** is left as is.
1685**
1686** If an error occurs, an SQLite error code is returned. Otherwise,
1687** SQLITE_OK.
1688*/
1689static int whereRangeSkipScanEst(
1690 Parse *pParse, /* Parsing & code generating context */
1691 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
1692 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
1693 WhereLoop *pLoop, /* Update the .nOut value of this loop */
1694 int *pbDone /* Set to true if at least one expr. value extracted */
1695){
1696 Index *p = pLoop->u.btree.pIndex;
1697 int nEq = pLoop->u.btree.nEq;
1698 sqlite3 *db = pParse->db;
1699 int nLower = -1;
1700 int nUpper = p->nSample+1;
1701 int rc = SQLITE_OK;
1702 u8 aff = sqlite3IndexColumnAffinity(db, p, nEq);
1703 CollSeq *pColl;
1704
1705 sqlite3_value *p1 = 0; /* Value extracted from pLower */
1706 sqlite3_value *p2 = 0; /* Value extracted from pUpper */
1707 sqlite3_value *pVal = 0; /* Value extracted from record */
1708
1709 pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]);
1710 if( pLower ){
1711 rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1);
1712 nLower = 0;
1713 }
1714 if( pUpper && rc==SQLITE_OK ){
1715 rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2);
1716 nUpper = p2 ? 0 : p->nSample;
1717 }
1718
1719 if( p1 || p2 ){
1720 int i;
1721 int nDiff;
1722 for(i=0; rc==SQLITE_OK && i<p->nSample; i++){
1723 rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal);
1724 if( rc==SQLITE_OK && p1 ){
1725 int res = sqlite3MemCompare(p1, pVal, pColl);
1726 if( res>=0 ) nLower++;
1727 }
1728 if( rc==SQLITE_OK && p2 ){
1729 int res = sqlite3MemCompare(p2, pVal, pColl);
1730 if( res>=0 ) nUpper++;
1731 }
1732 }
1733 nDiff = (nUpper - nLower);
1734 if( nDiff<=0 ) nDiff = 1;
1735
1736 /* If there is both an upper and lower bound specified, and the
1737 ** comparisons indicate that they are close together, use the fallback
1738 ** method (assume that the scan visits 1/64 of the rows) for estimating
1739 ** the number of rows visited. Otherwise, estimate the number of rows
1740 ** using the method described in the header comment for this function. */
1741 if( nDiff!=1 || pUpper==0 || pLower==0 ){
1742 int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff));
1743 pLoop->nOut -= nAdjust;
1744 *pbDone = 1;
1745 WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n",
1746 nLower, nUpper, nAdjust*-1, pLoop->nOut));
1747 }
1748
1749 }else{
1750 assert( *pbDone==0 );
1751 }
1752
1753 sqlite3ValueFree(p1);
1754 sqlite3ValueFree(p2);
1755 sqlite3ValueFree(pVal);
1756
1757 return rc;
1758}
1759#endif /* SQLITE_ENABLE_STAT4 */
1760
1761/*
1762** This function is used to estimate the number of rows that will be visited
1763** by scanning an index for a range of values. The range may have an upper
1764** bound, a lower bound, or both. The WHERE clause terms that set the upper
1765** and lower bounds are represented by pLower and pUpper respectively. For
1766** example, assuming that index p is on t1(a):
1767**
1768** ... FROM t1 WHERE a > ? AND a < ? ...
1769** |_____| |_____|
1770** | |
1771** pLower pUpper
1772**
1773** If either of the upper or lower bound is not present, then NULL is passed in
1774** place of the corresponding WhereTerm.
1775**
1776** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index
1777** column subject to the range constraint. Or, equivalently, the number of
1778** equality constraints optimized by the proposed index scan. For example,
1779** assuming index p is on t1(a, b), and the SQL query is:
1780**
1781** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
1782**
1783** then nEq is set to 1 (as the range restricted column, b, is the second
1784** left-most column of the index). Or, if the query is:
1785**
1786** ... FROM t1 WHERE a > ? AND a < ? ...
1787**
1788** then nEq is set to 0.
1789**
1790** When this function is called, *pnOut is set to the sqlite3LogEst() of the
1791** number of rows that the index scan is expected to visit without
1792** considering the range constraints. If nEq is 0, then *pnOut is the number of
1793** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
1794** to account for the range constraints pLower and pUpper.
1795**
1796** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
1797** used, a single range inequality reduces the search space by a factor of 4.
1798** and a pair of constraints (x>? AND x<?) reduces the expected number of
1799** rows visited by a factor of 64.
1800*/
1801static int whereRangeScanEst(
1802 Parse *pParse, /* Parsing & code generating context */
1803 WhereLoopBuilder *pBuilder,
1804 WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */
1805 WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */
1806 WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */
1807){
1808 int rc = SQLITE_OK;
1809 int nOut = pLoop->nOut;
1810 LogEst nNew;
1811
1812#ifdef SQLITE_ENABLE_STAT4
1813 Index *p = pLoop->u.btree.pIndex;
1814 int nEq = pLoop->u.btree.nEq;
1815
1816 if( p->nSample>0 && ALWAYS(nEq<p->nSampleCol)
1817 && OptimizationEnabled(pParse->db, SQLITE_Stat4)
1818 ){
1819 if( nEq==pBuilder->nRecValid ){
1820 UnpackedRecord *pRec = pBuilder->pRec;
1821 tRowcnt a[2];
1822 int nBtm = pLoop->u.btree.nBtm;
1823 int nTop = pLoop->u.btree.nTop;
1824
1825 /* Variable iLower will be set to the estimate of the number of rows in
1826 ** the index that are less than the lower bound of the range query. The
1827 ** lower bound being the concatenation of $P and $L, where $P is the
1828 ** key-prefix formed by the nEq values matched against the nEq left-most
1829 ** columns of the index, and $L is the value in pLower.
1830 **
1831 ** Or, if pLower is NULL or $L cannot be extracted from it (because it
1832 ** is not a simple variable or literal value), the lower bound of the
1833 ** range is $P. Due to a quirk in the way whereKeyStats() works, even
1834 ** if $L is available, whereKeyStats() is called for both ($P) and
1835 ** ($P:$L) and the larger of the two returned values is used.
1836 **
1837 ** Similarly, iUpper is to be set to the estimate of the number of rows
1838 ** less than the upper bound of the range query. Where the upper bound
1839 ** is either ($P) or ($P:$U). Again, even if $U is available, both values
1840 ** of iUpper are requested of whereKeyStats() and the smaller used.
1841 **
1842 ** The number of rows between the two bounds is then just iUpper-iLower.
1843 */
1844 tRowcnt iLower; /* Rows less than the lower bound */
1845 tRowcnt iUpper; /* Rows less than the upper bound */
1846 int iLwrIdx = -2; /* aSample[] for the lower bound */
1847 int iUprIdx = -1; /* aSample[] for the upper bound */
1848
1849 if( pRec ){
1850 testcase( pRec->nField!=pBuilder->nRecValid );
1851 pRec->nField = pBuilder->nRecValid;
1852 }
1853 /* Determine iLower and iUpper using ($P) only. */
1854 if( nEq==0 ){
1855 iLower = 0;
1856 iUpper = p->nRowEst0;
1857 }else{
1858 /* Note: this call could be optimized away - since the same values must
1859 ** have been requested when testing key $P in whereEqualScanEst(). */
1860 whereKeyStats(pParse, p, pRec, 0, a);
1861 iLower = a[0];
1862 iUpper = a[0] + a[1];
1863 }
1864
1865 assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
1866 assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
1867 assert( p->aSortOrder!=0 );
1868 if( p->aSortOrder[nEq] ){
1869 /* The roles of pLower and pUpper are swapped for a DESC index */
1870 SWAP(WhereTerm*, pLower, pUpper);
1871 SWAP(int, nBtm, nTop);
1872 }
1873
1874 /* If possible, improve on the iLower estimate using ($P:$L). */
1875 if( pLower ){
1876 int n; /* Values extracted from pExpr */
1877 Expr *pExpr = pLower->pExpr->pRight;
1878 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
1879 if( rc==SQLITE_OK && n ){
1880 tRowcnt iNew;
1881 u16 mask = WO_GT|WO_LE;
1882 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
1883 iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
1884 iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
1885 if( iNew>iLower ) iLower = iNew;
1886 nOut--;
1887 pLower = 0;
1888 }
1889 }
1890
1891 /* If possible, improve on the iUpper estimate using ($P:$U). */
1892 if( pUpper ){
1893 int n; /* Values extracted from pExpr */
1894 Expr *pExpr = pUpper->pExpr->pRight;
1895 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
1896 if( rc==SQLITE_OK && n ){
1897 tRowcnt iNew;
1898 u16 mask = WO_GT|WO_LE;
1899 if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
1900 iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
1901 iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
1902 if( iNew<iUpper ) iUpper = iNew;
1903 nOut--;
1904 pUpper = 0;
1905 }
1906 }
1907
1908 pBuilder->pRec = pRec;
1909 if( rc==SQLITE_OK ){
1910 if( iUpper>iLower ){
1911 nNew = sqlite3LogEst(iUpper - iLower);
1912 /* TUNING: If both iUpper and iLower are derived from the same
1913 ** sample, then assume they are 4x more selective. This brings
1914 ** the estimated selectivity more in line with what it would be
1915 ** if estimated without the use of STAT4 tables. */
1916 if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) );
1917 }else{
1918 nNew = 10; assert( 10==sqlite3LogEst(2) );
1919 }
1920 if( nNew<nOut ){
1921 nOut = nNew;
1922 }
1923 WHERETRACE(0x10, ("STAT4 range scan: %u..%u est=%d\n",
1924 (u32)iLower, (u32)iUpper, nOut));
1925 }
1926 }else{
1927 int bDone = 0;
1928 rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone);
1929 if( bDone ) return rc;
1930 }
1931 }
1932#else
1933 UNUSED_PARAMETER(pParse);
1934 UNUSED_PARAMETER(pBuilder);
1935 assert( pLower || pUpper );
1936#endif
1937 assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 );
1938 nNew = whereRangeAdjust(pLower, nOut);
1939 nNew = whereRangeAdjust(pUpper, nNew);
1940
1941 /* TUNING: If there is both an upper and lower limit and neither limit
1942 ** has an application-defined likelihood(), assume the range is
1943 ** reduced by an additional 75%. This means that, by default, an open-ended
1944 ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
1945 ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
1946 ** match 1/64 of the index. */
1947 if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
1948 nNew -= 20;
1949 }
1950
1951 nOut -= (pLower!=0) + (pUpper!=0);
1952 if( nNew<10 ) nNew = 10;
1953 if( nNew<nOut ) nOut = nNew;
1954#if defined(WHERETRACE_ENABLED)
1955 if( pLoop->nOut>nOut ){
1956 WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n",
1957 pLoop->nOut, nOut));
1958 }
1959#endif
1960 pLoop->nOut = (LogEst)nOut;
1961 return rc;
1962}
1963
1964#ifdef SQLITE_ENABLE_STAT4
1965/*
1966** Estimate the number of rows that will be returned based on
1967** an equality constraint x=VALUE and where that VALUE occurs in
1968** the histogram data. This only works when x is the left-most
1969** column of an index and sqlite_stat4 histogram data is available
1970** for that index. When pExpr==NULL that means the constraint is
1971** "x IS NULL" instead of "x=VALUE".
1972**
1973** Write the estimated row count into *pnRow and return SQLITE_OK.
1974** If unable to make an estimate, leave *pnRow unchanged and return
1975** non-zero.
1976**
1977** This routine can fail if it is unable to load a collating sequence
1978** required for string comparison, or if unable to allocate memory
1979** for a UTF conversion required for comparison. The error is stored
1980** in the pParse structure.
1981*/
1982static int whereEqualScanEst(
1983 Parse *pParse, /* Parsing & code generating context */
1984 WhereLoopBuilder *pBuilder,
1985 Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */
1986 tRowcnt *pnRow /* Write the revised row estimate here */
1987){
1988 Index *p = pBuilder->pNew->u.btree.pIndex;
1989 int nEq = pBuilder->pNew->u.btree.nEq;
1990 UnpackedRecord *pRec = pBuilder->pRec;
1991 int rc; /* Subfunction return code */
1992 tRowcnt a[2]; /* Statistics */
1993 int bOk;
1994
1995 assert( nEq>=1 );
1996 assert( nEq<=p->nColumn );
1997 assert( p->aSample!=0 );
1998 assert( p->nSample>0 );
1999 assert( pBuilder->nRecValid<nEq );
2000
2001 /* If values are not available for all fields of the index to the left
2002 ** of this one, no estimate can be made. Return SQLITE_NOTFOUND. */
2003 if( pBuilder->nRecValid<(nEq-1) ){
2004 return SQLITE_NOTFOUND;
2005 }
2006
2007 /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue()
2008 ** below would return the same value. */
2009 if( nEq>=p->nColumn ){
2010 *pnRow = 1;
2011 return SQLITE_OK;
2012 }
2013
2014 rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
2015 pBuilder->pRec = pRec;
2016 if( rc!=SQLITE_OK ) return rc;
2017 if( bOk==0 ) return SQLITE_NOTFOUND;
2018 pBuilder->nRecValid = nEq;
2019
2020 whereKeyStats(pParse, p, pRec, 0, a);
2021 WHERETRACE(0x10,("equality scan regions %s(%d): %d\n",
2022 p->zName, nEq-1, (int)a[1]));
2023 *pnRow = a[1];
2024
2025 return rc;
2026}
2027#endif /* SQLITE_ENABLE_STAT4 */
2028
2029#ifdef SQLITE_ENABLE_STAT4
2030/*
2031** Estimate the number of rows that will be returned based on
2032** an IN constraint where the right-hand side of the IN operator
2033** is a list of values. Example:
2034**
2035** WHERE x IN (1,2,3,4)
2036**
2037** Write the estimated row count into *pnRow and return SQLITE_OK.
2038** If unable to make an estimate, leave *pnRow unchanged and return
2039** non-zero.
2040**
2041** This routine can fail if it is unable to load a collating sequence
2042** required for string comparison, or if unable to allocate memory
2043** for a UTF conversion required for comparison. The error is stored
2044** in the pParse structure.
2045*/
2046static int whereInScanEst(
2047 Parse *pParse, /* Parsing & code generating context */
2048 WhereLoopBuilder *pBuilder,
2049 ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
2050 tRowcnt *pnRow /* Write the revised row estimate here */
2051){
2052 Index *p = pBuilder->pNew->u.btree.pIndex;
2053 i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]);
2054 int nRecValid = pBuilder->nRecValid;
2055 int rc = SQLITE_OK; /* Subfunction return code */
2056 tRowcnt nEst; /* Number of rows for a single term */
2057 tRowcnt nRowEst = 0; /* New estimate of the number of rows */
2058 int i; /* Loop counter */
2059
2060 assert( p->aSample!=0 );
2061 for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
2062 nEst = nRow0;
2063 rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst);
2064 nRowEst += nEst;
2065 pBuilder->nRecValid = nRecValid;
2066 }
2067
2068 if( rc==SQLITE_OK ){
2069 if( nRowEst > nRow0 ) nRowEst = nRow0;
2070 *pnRow = nRowEst;
2071 WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst));
2072 }
2073 assert( pBuilder->nRecValid==nRecValid );
2074 return rc;
2075}
2076#endif /* SQLITE_ENABLE_STAT4 */
2077
2078
2079#ifdef WHERETRACE_ENABLED
2080/*
2081** Print the content of a WhereTerm object
2082*/
2083void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
2084 if( pTerm==0 ){
2085 sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
2086 }else{
2087 char zType[8];
2088 char zLeft[50];
2089 memcpy(zType, "....", 5);
2090 if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
2091 if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E';
2092 if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L';
2093 if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C';
2094 if( pTerm->eOperator & WO_SINGLE ){
2095 assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
2096 sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
2097 pTerm->leftCursor, pTerm->u.x.leftColumn);
2098 }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
2099 sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx",
2100 pTerm->u.pOrInfo->indexable);
2101 }else{
2102 sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
2103 }
2104 sqlite3DebugPrintf(
2105 "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x",
2106 iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags);
2107 /* The 0x10000 .wheretrace flag causes extra information to be
2108 ** shown about each Term */
2109 if( sqlite3WhereTrace & 0x10000 ){
2110 sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx",
2111 pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight);
2112 }
2113 if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){
2114 sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField);
2115 }
2116 if( pTerm->iParent>=0 ){
2117 sqlite3DebugPrintf(" iParent=%d", pTerm->iParent);
2118 }
2119 sqlite3DebugPrintf("\n");
2120 sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
2121 }
2122}
2123#endif
2124
2125#ifdef WHERETRACE_ENABLED
2126/*
2127** Show the complete content of a WhereClause
2128*/
2129void sqlite3WhereClausePrint(WhereClause *pWC){
2130 int i;
2131 for(i=0; i<pWC->nTerm; i++){
2132 sqlite3WhereTermPrint(&pWC->a[i], i);
2133 }
2134}
2135#endif
2136
2137#ifdef WHERETRACE_ENABLED
2138/*
2139** Print a WhereLoop object for debugging purposes
2140*/
2141void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){
2142 WhereInfo *pWInfo = pWC->pWInfo;
2143 int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
2144 SrcItem *pItem = pWInfo->pTabList->a + p->iTab;
2145 Table *pTab = pItem->pTab;
2146 Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
2147 sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
2148 p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
2149 sqlite3DebugPrintf(" %12s",
2150 pItem->zAlias ? pItem->zAlias : pTab->zName);
2151 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
2152 const char *zName;
2153 if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){
2154 if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){
2155 int i = sqlite3Strlen30(zName) - 1;
2156 while( zName[i]!='_' ) i--;
2157 zName += i;
2158 }
2159 sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq);
2160 }else{
2161 sqlite3DebugPrintf("%20s","");
2162 }
2163 }else{
2164 char *z;
2165 if( p->u.vtab.idxStr ){
2166 z = sqlite3_mprintf("(%d,\"%s\",%#x)",
2167 p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask);
2168 }else{
2169 z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask);
2170 }
2171 sqlite3DebugPrintf(" %-19s", z);
2172 sqlite3_free(z);
2173 }
2174 if( p->wsFlags & WHERE_SKIPSCAN ){
2175 sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
2176 }else{
2177 sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
2178 }
2179 sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
2180 if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){
2181 int i;
2182 for(i=0; i<p->nLTerm; i++){
2183 sqlite3WhereTermPrint(p->aLTerm[i], i);
2184 }
2185 }
2186}
2187#endif
2188
2189/*
2190** Convert bulk memory into a valid WhereLoop that can be passed
2191** to whereLoopClear harmlessly.
2192*/
2193static void whereLoopInit(WhereLoop *p){
2194 p->aLTerm = p->aLTermSpace;
2195 p->nLTerm = 0;
2196 p->nLSlot = ArraySize(p->aLTermSpace);
2197 p->wsFlags = 0;
2198}
2199
2200/*
2201** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact.
2202*/
2203static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){
2204 if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){
2205 if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){
2206 sqlite3_free(p->u.vtab.idxStr);
2207 p->u.vtab.needFree = 0;
2208 p->u.vtab.idxStr = 0;
2209 }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){
2210 sqlite3DbFree(db, p->u.btree.pIndex->zColAff);
2211 sqlite3DbFreeNN(db, p->u.btree.pIndex);
2212 p->u.btree.pIndex = 0;
2213 }
2214 }
2215}
2216
2217/*
2218** Deallocate internal memory used by a WhereLoop object. Leave the
2219** object in an initialized state, as if it had been newly allocated.
2220*/
2221static void whereLoopClear(sqlite3 *db, WhereLoop *p){
2222 if( p->aLTerm!=p->aLTermSpace ){
2223 sqlite3DbFreeNN(db, p->aLTerm);
2224 p->aLTerm = p->aLTermSpace;
2225 p->nLSlot = ArraySize(p->aLTermSpace);
2226 }
2227 whereLoopClearUnion(db, p);
2228 p->nLTerm = 0;
2229 p->wsFlags = 0;
2230}
2231
2232/*
2233** Increase the memory allocation for pLoop->aLTerm[] to be at least n.
2234*/
2235static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
2236 WhereTerm **paNew;
2237 if( p->nLSlot>=n ) return SQLITE_OK;
2238 n = (n+7)&~7;
2239 paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n);
2240 if( paNew==0 ) return SQLITE_NOMEM_BKPT;
2241 memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
2242 if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm);
2243 p->aLTerm = paNew;
2244 p->nLSlot = n;
2245 return SQLITE_OK;
2246}
2247
2248/*
2249** Transfer content from the second pLoop into the first.
2250*/
2251static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
2252 whereLoopClearUnion(db, pTo);
2253 if( pFrom->nLTerm > pTo->nLSlot
2254 && whereLoopResize(db, pTo, pFrom->nLTerm)
2255 ){
2256 memset(pTo, 0, WHERE_LOOP_XFER_SZ);
2257 return SQLITE_NOMEM_BKPT;
2258 }
2259 memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
2260 memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
2261 if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){
2262 pFrom->u.vtab.needFree = 0;
2263 }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){
2264 pFrom->u.btree.pIndex = 0;
2265 }
2266 return SQLITE_OK;
2267}
2268
2269/*
2270** Delete a WhereLoop object
2271*/
2272static void whereLoopDelete(sqlite3 *db, WhereLoop *p){
2273 assert( db!=0 );
2274 whereLoopClear(db, p);
2275 sqlite3DbNNFreeNN(db, p);
2276}
2277
2278/*
2279** Free a WhereInfo structure
2280*/
2281static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
2282 assert( pWInfo!=0 );
2283 assert( db!=0 );
2284 sqlite3WhereClauseClear(&pWInfo->sWC);
2285 while( pWInfo->pLoops ){
2286 WhereLoop *p = pWInfo->pLoops;
2287 pWInfo->pLoops = p->pNextLoop;
2288 whereLoopDelete(db, p);
2289 }
2290 while( pWInfo->pMemToFree ){
2291 WhereMemBlock *pNext = pWInfo->pMemToFree->pNext;
2292 sqlite3DbNNFreeNN(db, pWInfo->pMemToFree);
2293 pWInfo->pMemToFree = pNext;
2294 }
2295 sqlite3DbNNFreeNN(db, pWInfo);
2296}
2297
2298/*
2299** Return TRUE if all of the following are true:
2300**
2301** (1) X has the same or lower cost, or returns the same or fewer rows,
2302** than Y.
2303** (2) X uses fewer WHERE clause terms than Y
2304** (3) Every WHERE clause term used by X is also used by Y
2305** (4) X skips at least as many columns as Y
2306** (5) If X is a covering index, than Y is too
2307**
2308** Conditions (2) and (3) mean that X is a "proper subset" of Y.
2309** If X is a proper subset of Y then Y is a better choice and ought
2310** to have a lower cost. This routine returns TRUE when that cost
2311** relationship is inverted and needs to be adjusted. Constraint (4)
2312** was added because if X uses skip-scan less than Y it still might
2313** deserve a lower cost even if it is a proper subset of Y. Constraint (5)
2314** was added because a covering index probably deserves to have a lower cost
2315** than a non-covering index even if it is a proper subset.
2316*/
2317static int whereLoopCheaperProperSubset(
2318 const WhereLoop *pX, /* First WhereLoop to compare */
2319 const WhereLoop *pY /* Compare against this WhereLoop */
2320){
2321 int i, j;
2322 if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
2323 return 0; /* X is not a subset of Y */
2324 }
2325 if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0;
2326 if( pY->nSkip > pX->nSkip ) return 0;
2327 for(i=pX->nLTerm-1; i>=0; i--){
2328 if( pX->aLTerm[i]==0 ) continue;
2329 for(j=pY->nLTerm-1; j>=0; j--){
2330 if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
2331 }
2332 if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */
2333 }
2334 if( (pX->wsFlags&WHERE_IDX_ONLY)!=0
2335 && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){
2336 return 0; /* Constraint (5) */
2337 }
2338 return 1; /* All conditions meet */
2339}
2340
2341/*
2342** Try to adjust the cost and number of output rows of WhereLoop pTemplate
2343** upwards or downwards so that:
2344**
2345** (1) pTemplate costs less than any other WhereLoops that are a proper
2346** subset of pTemplate
2347**
2348** (2) pTemplate costs more than any other WhereLoops for which pTemplate
2349** is a proper subset.
2350**
2351** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
2352** WHERE clause terms than Y and that every WHERE clause term used by X is
2353** also used by Y.
2354*/
2355static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
2356 if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
2357 for(; p; p=p->pNextLoop){
2358 if( p->iTab!=pTemplate->iTab ) continue;
2359 if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
2360 if( whereLoopCheaperProperSubset(p, pTemplate) ){
2361 /* Adjust pTemplate cost downward so that it is cheaper than its
2362 ** subset p. */
2363 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
2364 pTemplate->rRun, pTemplate->nOut,
2365 MIN(p->rRun, pTemplate->rRun),
2366 MIN(p->nOut - 1, pTemplate->nOut)));
2367 pTemplate->rRun = MIN(p->rRun, pTemplate->rRun);
2368 pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut);
2369 }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
2370 /* Adjust pTemplate cost upward so that it is costlier than p since
2371 ** pTemplate is a proper subset of p */
2372 WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
2373 pTemplate->rRun, pTemplate->nOut,
2374 MAX(p->rRun, pTemplate->rRun),
2375 MAX(p->nOut + 1, pTemplate->nOut)));
2376 pTemplate->rRun = MAX(p->rRun, pTemplate->rRun);
2377 pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut);
2378 }
2379 }
2380}
2381
2382/*
2383** Search the list of WhereLoops in *ppPrev looking for one that can be
2384** replaced by pTemplate.
2385**
2386** Return NULL if pTemplate does not belong on the WhereLoop list.
2387** In other words if pTemplate ought to be dropped from further consideration.
2388**
2389** If pX is a WhereLoop that pTemplate can replace, then return the
2390** link that points to pX.
2391**
2392** If pTemplate cannot replace any existing element of the list but needs
2393** to be added to the list as a new entry, then return a pointer to the
2394** tail of the list.
2395*/
2396static WhereLoop **whereLoopFindLesser(
2397 WhereLoop **ppPrev,
2398 const WhereLoop *pTemplate
2399){
2400 WhereLoop *p;
2401 for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){
2402 if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){
2403 /* If either the iTab or iSortIdx values for two WhereLoop are different
2404 ** then those WhereLoops need to be considered separately. Neither is
2405 ** a candidate to replace the other. */
2406 continue;
2407 }
2408 /* In the current implementation, the rSetup value is either zero
2409 ** or the cost of building an automatic index (NlogN) and the NlogN
2410 ** is the same for compatible WhereLoops. */
2411 assert( p->rSetup==0 || pTemplate->rSetup==0
2412 || p->rSetup==pTemplate->rSetup );
2413
2414 /* whereLoopAddBtree() always generates and inserts the automatic index
2415 ** case first. Hence compatible candidate WhereLoops never have a larger
2416 ** rSetup. Call this SETUP-INVARIANT */
2417 assert( p->rSetup>=pTemplate->rSetup );
2418
2419 /* Any loop using an appliation-defined index (or PRIMARY KEY or
2420 ** UNIQUE constraint) with one or more == constraints is better
2421 ** than an automatic index. Unless it is a skip-scan. */
2422 if( (p->wsFlags & WHERE_AUTO_INDEX)!=0
2423 && (pTemplate->nSkip)==0
2424 && (pTemplate->wsFlags & WHERE_INDEXED)!=0
2425 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0
2426 && (p->prereq & pTemplate->prereq)==pTemplate->prereq
2427 ){
2428 break;
2429 }
2430
2431 /* If existing WhereLoop p is better than pTemplate, pTemplate can be
2432 ** discarded. WhereLoop p is better if:
2433 ** (1) p has no more dependencies than pTemplate, and
2434 ** (2) p has an equal or lower cost than pTemplate
2435 */
2436 if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */
2437 && p->rSetup<=pTemplate->rSetup /* (2a) */
2438 && p->rRun<=pTemplate->rRun /* (2b) */
2439 && p->nOut<=pTemplate->nOut /* (2c) */
2440 ){
2441 return 0; /* Discard pTemplate */
2442 }
2443
2444 /* If pTemplate is always better than p, then cause p to be overwritten
2445 ** with pTemplate. pTemplate is better than p if:
2446 ** (1) pTemplate has no more dependences than p, and
2447 ** (2) pTemplate has an equal or lower cost than p.
2448 */
2449 if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */
2450 && p->rRun>=pTemplate->rRun /* (2a) */
2451 && p->nOut>=pTemplate->nOut /* (2b) */
2452 ){
2453 assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */
2454 break; /* Cause p to be overwritten by pTemplate */
2455 }
2456 }
2457 return ppPrev;
2458}
2459
2460/*
2461** Insert or replace a WhereLoop entry using the template supplied.
2462**
2463** An existing WhereLoop entry might be overwritten if the new template
2464** is better and has fewer dependencies. Or the template will be ignored
2465** and no insert will occur if an existing WhereLoop is faster and has
2466** fewer dependencies than the template. Otherwise a new WhereLoop is
2467** added based on the template.
2468**
2469** If pBuilder->pOrSet is not NULL then we care about only the
2470** prerequisites and rRun and nOut costs of the N best loops. That
2471** information is gathered in the pBuilder->pOrSet object. This special
2472** processing mode is used only for OR clause processing.
2473**
2474** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
2475** still might overwrite similar loops with the new template if the
2476** new template is better. Loops may be overwritten if the following
2477** conditions are met:
2478**
2479** (1) They have the same iTab.
2480** (2) They have the same iSortIdx.
2481** (3) The template has same or fewer dependencies than the current loop
2482** (4) The template has the same or lower cost than the current loop
2483*/
2484static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
2485 WhereLoop **ppPrev, *p;
2486 WhereInfo *pWInfo = pBuilder->pWInfo;
2487 sqlite3 *db = pWInfo->pParse->db;
2488 int rc;
2489
2490 /* Stop the search once we hit the query planner search limit */
2491 if( pBuilder->iPlanLimit==0 ){
2492 WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n"));
2493 if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0;
2494 return SQLITE_DONE;
2495 }
2496 pBuilder->iPlanLimit--;
2497
2498 whereLoopAdjustCost(pWInfo->pLoops, pTemplate);
2499
2500 /* If pBuilder->pOrSet is defined, then only keep track of the costs
2501 ** and prereqs.
2502 */
2503 if( pBuilder->pOrSet!=0 ){
2504 if( pTemplate->nLTerm ){
2505#if WHERETRACE_ENABLED
2506 u16 n = pBuilder->pOrSet->n;
2507 int x =
2508#endif
2509 whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun,
2510 pTemplate->nOut);
2511#if WHERETRACE_ENABLED /* 0x8 */
2512 if( sqlite3WhereTrace & 0x8 ){
2513 sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n);
2514 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
2515 }
2516#endif
2517 }
2518 return SQLITE_OK;
2519 }
2520
2521 /* Look for an existing WhereLoop to replace with pTemplate
2522 */
2523 ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate);
2524
2525 if( ppPrev==0 ){
2526 /* There already exists a WhereLoop on the list that is better
2527 ** than pTemplate, so just ignore pTemplate */
2528#if WHERETRACE_ENABLED /* 0x8 */
2529 if( sqlite3WhereTrace & 0x8 ){
2530 sqlite3DebugPrintf(" skip: ");
2531 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
2532 }
2533#endif
2534 return SQLITE_OK;
2535 }else{
2536 p = *ppPrev;
2537 }
2538
2539 /* If we reach this point it means that either p[] should be overwritten
2540 ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new
2541 ** WhereLoop and insert it.
2542 */
2543#if WHERETRACE_ENABLED /* 0x8 */
2544 if( sqlite3WhereTrace & 0x8 ){
2545 if( p!=0 ){
2546 sqlite3DebugPrintf("replace: ");
2547 sqlite3WhereLoopPrint(p, pBuilder->pWC);
2548 sqlite3DebugPrintf(" with: ");
2549 }else{
2550 sqlite3DebugPrintf(" add: ");
2551 }
2552 sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
2553 }
2554#endif
2555 if( p==0 ){
2556 /* Allocate a new WhereLoop to add to the end of the list */
2557 *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop));
2558 if( p==0 ) return SQLITE_NOMEM_BKPT;
2559 whereLoopInit(p);
2560 p->pNextLoop = 0;
2561 }else{
2562 /* We will be overwriting WhereLoop p[]. But before we do, first
2563 ** go through the rest of the list and delete any other entries besides
2564 ** p[] that are also supplated by pTemplate */
2565 WhereLoop **ppTail = &p->pNextLoop;
2566 WhereLoop *pToDel;
2567 while( *ppTail ){
2568 ppTail = whereLoopFindLesser(ppTail, pTemplate);
2569 if( ppTail==0 ) break;
2570 pToDel = *ppTail;
2571 if( pToDel==0 ) break;
2572 *ppTail = pToDel->pNextLoop;
2573#if WHERETRACE_ENABLED /* 0x8 */
2574 if( sqlite3WhereTrace & 0x8 ){
2575 sqlite3DebugPrintf(" delete: ");
2576 sqlite3WhereLoopPrint(pToDel, pBuilder->pWC);
2577 }
2578#endif
2579 whereLoopDelete(db, pToDel);
2580 }
2581 }
2582 rc = whereLoopXfer(db, p, pTemplate);
2583 if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
2584 Index *pIndex = p->u.btree.pIndex;
2585 if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){
2586 p->u.btree.pIndex = 0;
2587 }
2588 }
2589 return rc;
2590}
2591
2592/*
2593** Adjust the WhereLoop.nOut value downward to account for terms of the
2594** WHERE clause that reference the loop but which are not used by an
2595** index.
2596*
2597** For every WHERE clause term that is not used by the index
2598** and which has a truth probability assigned by one of the likelihood(),
2599** likely(), or unlikely() SQL functions, reduce the estimated number
2600** of output rows by the probability specified.
2601**
2602** TUNING: For every WHERE clause term that is not used by the index
2603** and which does not have an assigned truth probability, heuristics
2604** described below are used to try to estimate the truth probability.
2605** TODO --> Perhaps this is something that could be improved by better
2606** table statistics.
2607**
2608** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75%
2609** value corresponds to -1 in LogEst notation, so this means decrement
2610** the WhereLoop.nOut field for every such WHERE clause term.
2611**
2612** Heuristic 2: If there exists one or more WHERE clause terms of the
2613** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
2614** final output row estimate is no greater than 1/4 of the total number
2615** of rows in the table. In other words, assume that x==EXPR will filter
2616** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the
2617** "x" column is boolean or else -1 or 0 or 1 is a common default value
2618** on the "x" column and so in that case only cap the output row estimate
2619** at 1/2 instead of 1/4.
2620*/
2621static void whereLoopOutputAdjust(
2622 WhereClause *pWC, /* The WHERE clause */
2623 WhereLoop *pLoop, /* The loop to adjust downward */
2624 LogEst nRow /* Number of rows in the entire table */
2625){
2626 WhereTerm *pTerm, *pX;
2627 Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
2628 int i, j;
2629 LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */
2630
2631 assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
2632 for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){
2633 assert( pTerm!=0 );
2634 if( (pTerm->prereqAll & notAllowed)!=0 ) continue;
2635 if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue;
2636 if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue;
2637 for(j=pLoop->nLTerm-1; j>=0; j--){
2638 pX = pLoop->aLTerm[j];
2639 if( pX==0 ) continue;
2640 if( pX==pTerm ) break;
2641 if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break;
2642 }
2643 if( j<0 ){
2644 if( pLoop->maskSelf==pTerm->prereqAll ){
2645 /* If there are extra terms in the WHERE clause not used by an index
2646 ** that depend only on the table being scanned, and that will tend to
2647 ** cause many rows to be omitted, then mark that table as
2648 ** "self-culling".
2649 **
2650 ** 2022-03-24: Self-culling only applies if either the extra terms
2651 ** are straight comparison operators that are non-true with NULL
2652 ** operand, or if the loop is not an OUTER JOIN.
2653 */
2654 if( (pTerm->eOperator & 0x3f)!=0
2655 || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype
2656 & (JT_LEFT|JT_LTORJ))==0
2657 ){
2658 pLoop->wsFlags |= WHERE_SELFCULL;
2659 }
2660 }
2661 if( pTerm->truthProb<=0 ){
2662 /* If a truth probability is specified using the likelihood() hints,
2663 ** then use the probability provided by the application. */
2664 pLoop->nOut += pTerm->truthProb;
2665 }else{
2666 /* In the absence of explicit truth probabilities, use heuristics to
2667 ** guess a reasonable truth probability. */
2668 pLoop->nOut--;
2669 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0
2670 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */
2671 ){
2672 Expr *pRight = pTerm->pExpr->pRight;
2673 int k = 0;
2674 testcase( pTerm->pExpr->op==TK_IS );
2675 if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
2676 k = 10;
2677 }else{
2678 k = 20;
2679 }
2680 if( iReduce<k ){
2681 pTerm->wtFlags |= TERM_HEURTRUTH;
2682 iReduce = k;
2683 }
2684 }
2685 }
2686 }
2687 }
2688 if( pLoop->nOut > nRow-iReduce ){
2689 pLoop->nOut = nRow - iReduce;
2690 }
2691}
2692
2693/*
2694** Term pTerm is a vector range comparison operation. The first comparison
2695** in the vector can be optimized using column nEq of the index. This
2696** function returns the total number of vector elements that can be used
2697** as part of the range comparison.
2698**
2699** For example, if the query is:
2700**
2701** WHERE a = ? AND (b, c, d) > (?, ?, ?)
2702**
2703** and the index:
2704**
2705** CREATE INDEX ... ON (a, b, c, d, e)
2706**
2707** then this function would be invoked with nEq=1. The value returned in
2708** this case is 3.
2709*/
2710static int whereRangeVectorLen(
2711 Parse *pParse, /* Parsing context */
2712 int iCur, /* Cursor open on pIdx */
2713 Index *pIdx, /* The index to be used for a inequality constraint */
2714 int nEq, /* Number of prior equality constraints on same index */
2715 WhereTerm *pTerm /* The vector inequality constraint */
2716){
2717 int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
2718 int i;
2719
2720 nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
2721 for(i=1; i<nCmp; i++){
2722 /* Test if comparison i of pTerm is compatible with column (i+nEq)
2723 ** of the index. If not, exit the loop. */
2724 char aff; /* Comparison affinity */
2725 char idxaff = 0; /* Indexed columns affinity */
2726 CollSeq *pColl; /* Comparison collation sequence */
2727 Expr *pLhs, *pRhs;
2728
2729 assert( ExprUseXList(pTerm->pExpr->pLeft) );
2730 pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr;
2731 pRhs = pTerm->pExpr->pRight;
2732 if( ExprUseXSelect(pRhs) ){
2733 pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
2734 }else{
2735 pRhs = pRhs->x.pList->a[i].pExpr;
2736 }
2737
2738 /* Check that the LHS of the comparison is a column reference to
2739 ** the right column of the right source table. And that the sort
2740 ** order of the index column is the same as the sort order of the
2741 ** leftmost index column. */
2742 if( pLhs->op!=TK_COLUMN
2743 || pLhs->iTable!=iCur
2744 || pLhs->iColumn!=pIdx->aiColumn[i+nEq]
2745 || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
2746 ){
2747 break;
2748 }
2749
2750 testcase( pLhs->iColumn==XN_ROWID );
2751 aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
2752 idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
2753 if( aff!=idxaff ) break;
2754
2755 pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
2756 if( pColl==0 ) break;
2757 if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
2758 }
2759 return i;
2760}
2761
2762/*
2763** Adjust the cost C by the costMult facter T. This only occurs if
2764** compiled with -DSQLITE_ENABLE_COSTMULT
2765*/
2766#ifdef SQLITE_ENABLE_COSTMULT
2767# define ApplyCostMultiplier(C,T) C += T
2768#else
2769# define ApplyCostMultiplier(C,T)
2770#endif
2771
2772/*
2773** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
2774** index pIndex. Try to match one more.
2775**
2776** When this function is called, pBuilder->pNew->nOut contains the
2777** number of rows expected to be visited by filtering using the nEq
2778** terms only. If it is modified, this value is restored before this
2779** function returns.
2780**
2781** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is
2782** a fake index used for the INTEGER PRIMARY KEY.
2783*/
2784static int whereLoopAddBtreeIndex(
2785 WhereLoopBuilder *pBuilder, /* The WhereLoop factory */
2786 SrcItem *pSrc, /* FROM clause term being analyzed */
2787 Index *pProbe, /* An index on pSrc */
2788 LogEst nInMul /* log(Number of iterations due to IN) */
2789){
2790 WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */
2791 Parse *pParse = pWInfo->pParse; /* Parsing context */
2792 sqlite3 *db = pParse->db; /* Database connection malloc context */
2793 WhereLoop *pNew; /* Template WhereLoop under construction */
2794 WhereTerm *pTerm; /* A WhereTerm under consideration */
2795 int opMask; /* Valid operators for constraints */
2796 WhereScan scan; /* Iterator for WHERE terms */
2797 Bitmask saved_prereq; /* Original value of pNew->prereq */
2798 u16 saved_nLTerm; /* Original value of pNew->nLTerm */
2799 u16 saved_nEq; /* Original value of pNew->u.btree.nEq */
2800 u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */
2801 u16 saved_nTop; /* Original value of pNew->u.btree.nTop */
2802 u16 saved_nSkip; /* Original value of pNew->nSkip */
2803 u32 saved_wsFlags; /* Original value of pNew->wsFlags */
2804 LogEst saved_nOut; /* Original value of pNew->nOut */
2805 int rc = SQLITE_OK; /* Return code */
2806 LogEst rSize; /* Number of rows in the table */
2807 LogEst rLogSize; /* Logarithm of table size */
2808 WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
2809
2810 pNew = pBuilder->pNew;
2811 if( db->mallocFailed ) return SQLITE_NOMEM_BKPT;
2812 WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n",
2813 pProbe->pTable->zName,pProbe->zName,
2814 pNew->u.btree.nEq, pNew->nSkip, pNew->rRun));
2815
2816 assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
2817 assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
2818 if( pNew->wsFlags & WHERE_BTM_LIMIT ){
2819 opMask = WO_LT|WO_LE;
2820 }else{
2821 assert( pNew->u.btree.nBtm==0 );
2822 opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
2823 }
2824 if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
2825
2826 assert( pNew->u.btree.nEq<pProbe->nColumn );
2827 assert( pNew->u.btree.nEq<pProbe->nKeyCol
2828 || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY );
2829
2830 saved_nEq = pNew->u.btree.nEq;
2831 saved_nBtm = pNew->u.btree.nBtm;
2832 saved_nTop = pNew->u.btree.nTop;
2833 saved_nSkip = pNew->nSkip;
2834 saved_nLTerm = pNew->nLTerm;
2835 saved_wsFlags = pNew->wsFlags;
2836 saved_prereq = pNew->prereq;
2837 saved_nOut = pNew->nOut;
2838 pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq,
2839 opMask, pProbe);
2840 pNew->rSetup = 0;
2841 rSize = pProbe->aiRowLogEst[0];
2842 rLogSize = estLog(rSize);
2843 for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
2844 u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */
2845 LogEst rCostIdx;
2846 LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */
2847 int nIn = 0;
2848#ifdef SQLITE_ENABLE_STAT4
2849 int nRecValid = pBuilder->nRecValid;
2850#endif
2851 if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0)
2852 && indexColumnNotNull(pProbe, saved_nEq)
2853 ){
2854 continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */
2855 }
2856 if( pTerm->prereqRight & pNew->maskSelf ) continue;
2857
2858 /* Do not allow the upper bound of a LIKE optimization range constraint
2859 ** to mix with a lower range bound from some other source */
2860 if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
2861
2862 if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
2863 && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
2864 ){
2865 continue;
2866 }
2867 if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
2868 pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE;
2869 }else{
2870 pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED;
2871 }
2872 pNew->wsFlags = saved_wsFlags;
2873 pNew->u.btree.nEq = saved_nEq;
2874 pNew->u.btree.nBtm = saved_nBtm;
2875 pNew->u.btree.nTop = saved_nTop;
2876 pNew->nLTerm = saved_nLTerm;
2877 if( pNew->nLTerm>=pNew->nLSlot
2878 && whereLoopResize(db, pNew, pNew->nLTerm+1)
2879 ){
2880 break; /* OOM while trying to enlarge the pNew->aLTerm array */
2881 }
2882 pNew->aLTerm[pNew->nLTerm++] = pTerm;
2883 pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
2884
2885 assert( nInMul==0
2886 || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0
2887 || (pNew->wsFlags & WHERE_COLUMN_IN)!=0
2888 || (pNew->wsFlags & WHERE_SKIPSCAN)!=0
2889 );
2890
2891 if( eOp & WO_IN ){
2892 Expr *pExpr = pTerm->pExpr;
2893 if( ExprUseXSelect(pExpr) ){
2894 /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
2895 int i;
2896 nIn = 46; assert( 46==sqlite3LogEst(25) );
2897
2898 /* The expression may actually be of the form (x, y) IN (SELECT...).
2899 ** In this case there is a separate term for each of (x) and (y).
2900 ** However, the nIn multiplier should only be applied once, not once
2901 ** for each such term. The following loop checks that pTerm is the
2902 ** first such term in use, and sets nIn back to 0 if it is not. */
2903 for(i=0; i<pNew->nLTerm-1; i++){
2904 if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
2905 }
2906 }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
2907 /* "x IN (value, value, ...)" */
2908 nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
2909 }
2910 if( pProbe->hasStat1 && rLogSize>=10 ){
2911 LogEst M, logK, x;
2912 /* Let:
2913 ** N = the total number of rows in the table
2914 ** K = the number of entries on the RHS of the IN operator
2915 ** M = the number of rows in the table that match terms to the
2916 ** to the left in the same index. If the IN operator is on
2917 ** the left-most index column, M==N.
2918 **
2919 ** Given the definitions above, it is better to omit the IN operator
2920 ** from the index lookup and instead do a scan of the M elements,
2921 ** testing each scanned row against the IN operator separately, if:
2922 **
2923 ** M*log(K) < K*log(N)
2924 **
2925 ** Our estimates for M, K, and N might be inaccurate, so we build in
2926 ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
2927 ** with the index, as using an index has better worst-case behavior.
2928 ** If we do not have real sqlite_stat1 data, always prefer to use
2929 ** the index. Do not bother with this optimization on very small
2930 ** tables (less than 2 rows) as it is pointless in that case.
2931 */
2932 M = pProbe->aiRowLogEst[saved_nEq];
2933 logK = estLog(nIn);
2934 /* TUNING v----- 10 to bias toward indexed IN */
2935 x = M + logK + 10 - (nIn + rLogSize);
2936 if( x>=0 ){
2937 WHERETRACE(0x40,
2938 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
2939 "prefers indexed lookup\n",
2940 saved_nEq, M, logK, nIn, rLogSize, x));
2941 }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){
2942 WHERETRACE(0x40,
2943 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
2944 " nInMul=%d) prefers skip-scan\n",
2945 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
2946 pNew->wsFlags |= WHERE_IN_SEEKSCAN;
2947 }else{
2948 WHERETRACE(0x40,
2949 ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
2950 " nInMul=%d) prefers normal scan\n",
2951 saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
2952 continue;
2953 }
2954 }
2955 pNew->wsFlags |= WHERE_COLUMN_IN;
2956 }else if( eOp & (WO_EQ|WO_IS) ){
2957 int iCol = pProbe->aiColumn[saved_nEq];
2958 pNew->wsFlags |= WHERE_COLUMN_EQ;
2959 assert( saved_nEq==pNew->u.btree.nEq );
2960 if( iCol==XN_ROWID
2961 || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
2962 ){
2963 if( iCol==XN_ROWID || pProbe->uniqNotNull
2964 || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ)
2965 ){
2966 pNew->wsFlags |= WHERE_ONEROW;
2967 }else{
2968 pNew->wsFlags |= WHERE_UNQ_WANTED;
2969 }
2970 }
2971 if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS;
2972 }else if( eOp & WO_ISNULL ){
2973 pNew->wsFlags |= WHERE_COLUMN_NULL;
2974 }else{
2975 int nVecLen = whereRangeVectorLen(
2976 pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
2977 );
2978 if( eOp & (WO_GT|WO_GE) ){
2979 testcase( eOp & WO_GT );
2980 testcase( eOp & WO_GE );
2981 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
2982 pNew->u.btree.nBtm = nVecLen;
2983 pBtm = pTerm;
2984 pTop = 0;
2985 if( pTerm->wtFlags & TERM_LIKEOPT ){
2986 /* Range constraints that come from the LIKE optimization are
2987 ** always used in pairs. */
2988 pTop = &pTerm[1];
2989 assert( (pTop-(pTerm->pWC->a))<pTerm->pWC->nTerm );
2990 assert( pTop->wtFlags & TERM_LIKEOPT );
2991 assert( pTop->eOperator==WO_LT );
2992 if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
2993 pNew->aLTerm[pNew->nLTerm++] = pTop;
2994 pNew->wsFlags |= WHERE_TOP_LIMIT;
2995 pNew->u.btree.nTop = 1;
2996 }
2997 }else{
2998 assert( eOp & (WO_LT|WO_LE) );
2999 testcase( eOp & WO_LT );
3000 testcase( eOp & WO_LE );
3001 pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
3002 pNew->u.btree.nTop = nVecLen;
3003 pTop = pTerm;
3004 pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
3005 pNew->aLTerm[pNew->nLTerm-2] : 0;
3006 }
3007 }
3008
3009 /* At this point pNew->nOut is set to the number of rows expected to
3010 ** be visited by the index scan before considering term pTerm, or the
3011 ** values of nIn and nInMul. In other words, assuming that all
3012 ** "x IN(...)" terms are replaced with "x = ?". This block updates
3013 ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */
3014 assert( pNew->nOut==saved_nOut );
3015 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
3016 /* Adjust nOut using stat4 data. Or, if there is no stat4
3017 ** data, using some other estimate. */
3018 whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew);
3019 }else{
3020 int nEq = ++pNew->u.btree.nEq;
3021 assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) );
3022
3023 assert( pNew->nOut==saved_nOut );
3024 if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){
3025 assert( (eOp & WO_IN) || nIn==0 );
3026 testcase( eOp & WO_IN );
3027 pNew->nOut += pTerm->truthProb;
3028 pNew->nOut -= nIn;
3029 }else{
3030#ifdef SQLITE_ENABLE_STAT4
3031 tRowcnt nOut = 0;
3032 if( nInMul==0
3033 && pProbe->nSample
3034 && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol)
3035 && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr))
3036 && OptimizationEnabled(db, SQLITE_Stat4)
3037 ){
3038 Expr *pExpr = pTerm->pExpr;
3039 if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){
3040 testcase( eOp & WO_EQ );
3041 testcase( eOp & WO_IS );
3042 testcase( eOp & WO_ISNULL );
3043 rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut);
3044 }else{
3045 rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut);
3046 }
3047 if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
3048 if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */
3049 if( nOut ){
3050 pNew->nOut = sqlite3LogEst(nOut);
3051 if( nEq==1
3052 /* TUNING: Mark terms as "low selectivity" if they seem likely
3053 ** to be true for half or more of the rows in the table.
3054 ** See tag-202002240-1 */
3055 && pNew->nOut+10 > pProbe->aiRowLogEst[0]
3056 ){
3057#if WHERETRACE_ENABLED /* 0x01 */
3058 if( sqlite3WhereTrace & 0x01 ){
3059 sqlite3DebugPrintf(
3060 "STAT4 determines term has low selectivity:\n");
3061 sqlite3WhereTermPrint(pTerm, 999);
3062 }
3063#endif
3064 pTerm->wtFlags |= TERM_HIGHTRUTH;
3065 if( pTerm->wtFlags & TERM_HEURTRUTH ){
3066 /* If the term has previously been used with an assumption of
3067 ** higher selectivity, then set the flag to rerun the
3068 ** loop computations. */
3069 pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS;
3070 }
3071 }
3072 if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut;
3073 pNew->nOut -= nIn;
3074 }
3075 }
3076 if( nOut==0 )
3077#endif
3078 {
3079 pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]);
3080 if( eOp & WO_ISNULL ){
3081 /* TUNING: If there is no likelihood() value, assume that a
3082 ** "col IS NULL" expression matches twice as many rows
3083 ** as (col=?). */
3084 pNew->nOut += 10;
3085 }
3086 }
3087 }
3088 }
3089
3090 /* Set rCostIdx to the cost of visiting selected rows in index. Add
3091 ** it to pNew->rRun, which is currently set to the cost of the index
3092 ** seek only. Then, if this is a non-covering index, add the cost of
3093 ** visiting the rows in the main table. */
3094 assert( pSrc->pTab->szTabRow>0 );
3095 rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow;
3096 pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx);
3097 if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){
3098 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
3099 }
3100 ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult);
3101
3102 nOutUnadjusted = pNew->nOut;
3103 pNew->rRun += nInMul + nIn;
3104 pNew->nOut += nInMul + nIn;
3105 whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize);
3106 rc = whereLoopInsert(pBuilder, pNew);
3107
3108 if( pNew->wsFlags & WHERE_COLUMN_RANGE ){
3109 pNew->nOut = saved_nOut;
3110 }else{
3111 pNew->nOut = nOutUnadjusted;
3112 }
3113
3114 if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
3115 && pNew->u.btree.nEq<pProbe->nColumn
3116 && (pNew->u.btree.nEq<pProbe->nKeyCol ||
3117 pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
3118 ){
3119 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn);
3120 }
3121 pNew->nOut = saved_nOut;
3122#ifdef SQLITE_ENABLE_STAT4
3123 pBuilder->nRecValid = nRecValid;
3124#endif
3125 }
3126 pNew->prereq = saved_prereq;
3127 pNew->u.btree.nEq = saved_nEq;
3128 pNew->u.btree.nBtm = saved_nBtm;
3129 pNew->u.btree.nTop = saved_nTop;
3130 pNew->nSkip = saved_nSkip;
3131 pNew->wsFlags = saved_wsFlags;
3132 pNew->nOut = saved_nOut;
3133 pNew->nLTerm = saved_nLTerm;
3134
3135 /* Consider using a skip-scan if there are no WHERE clause constraints
3136 ** available for the left-most terms of the index, and if the average
3137 ** number of repeats in the left-most terms is at least 18.
3138 **
3139 ** The magic number 18 is selected on the basis that scanning 17 rows
3140 ** is almost always quicker than an index seek (even though if the index
3141 ** contains fewer than 2^17 rows we assume otherwise in other parts of
3142 ** the code). And, even if it is not, it should not be too much slower.
3143 ** On the other hand, the extra seeks could end up being significantly
3144 ** more expensive. */
3145 assert( 42==sqlite3LogEst(18) );
3146 if( saved_nEq==saved_nSkip
3147 && saved_nEq+1<pProbe->nKeyCol
3148 && saved_nEq==pNew->nLTerm
3149 && pProbe->noSkipScan==0
3150 && pProbe->hasStat1!=0
3151 && OptimizationEnabled(db, SQLITE_SkipScan)
3152 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */
3153 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
3154 ){
3155 LogEst nIter;
3156 pNew->u.btree.nEq++;
3157 pNew->nSkip++;
3158 pNew->aLTerm[pNew->nLTerm++] = 0;
3159 pNew->wsFlags |= WHERE_SKIPSCAN;
3160 nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
3161 pNew->nOut -= nIter;
3162 /* TUNING: Because uncertainties in the estimates for skip-scan queries,
3163 ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
3164 nIter += 5;
3165 whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
3166 pNew->nOut = saved_nOut;
3167 pNew->u.btree.nEq = saved_nEq;
3168 pNew->nSkip = saved_nSkip;
3169 pNew->wsFlags = saved_wsFlags;
3170 }
3171
3172 WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n",
3173 pProbe->pTable->zName, pProbe->zName, saved_nEq, rc));
3174 return rc;
3175}
3176
3177/*
3178** Return True if it is possible that pIndex might be useful in
3179** implementing the ORDER BY clause in pBuilder.
3180**
3181** Return False if pBuilder does not contain an ORDER BY clause or
3182** if there is no way for pIndex to be useful in implementing that
3183** ORDER BY clause.
3184*/
3185static int indexMightHelpWithOrderBy(
3186 WhereLoopBuilder *pBuilder,
3187 Index *pIndex,
3188 int iCursor
3189){
3190 ExprList *pOB;
3191 ExprList *aColExpr;
3192 int ii, jj;
3193
3194 if( pIndex->bUnordered ) return 0;
3195 if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0;
3196 for(ii=0; ii<pOB->nExpr; ii++){
3197 Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
3198 if( NEVER(pExpr==0) ) continue;
3199 if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){
3200 if( pExpr->iColumn<0 ) return 1;
3201 for(jj=0; jj<pIndex->nKeyCol; jj++){
3202 if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
3203 }
3204 }else if( (aColExpr = pIndex->aColExpr)!=0 ){
3205 for(jj=0; jj<pIndex->nKeyCol; jj++){
3206 if( pIndex->aiColumn[jj]!=XN_EXPR ) continue;
3207 if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){
3208 return 1;
3209 }
3210 }
3211 }
3212 }
3213 return 0;
3214}
3215
3216/* Check to see if a partial index with pPartIndexWhere can be used
3217** in the current query. Return true if it can be and false if not.
3218*/
3219static int whereUsablePartialIndex(
3220 int iTab, /* The table for which we want an index */
3221 u8 jointype, /* The JT_* flags on the join */
3222 WhereClause *pWC, /* The WHERE clause of the query */
3223 Expr *pWhere /* The WHERE clause from the partial index */
3224){
3225 int i;
3226 WhereTerm *pTerm;
3227 Parse *pParse;
3228
3229 if( jointype & JT_LTORJ ) return 0;
3230 pParse = pWC->pWInfo->pParse;
3231 while( pWhere->op==TK_AND ){
3232 if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0;
3233 pWhere = pWhere->pRight;
3234 }
3235 if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0;
3236 for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
3237 Expr *pExpr;
3238 pExpr = pTerm->pExpr;
3239 if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab)
3240 && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON))
3241 && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab)
3242 && (pTerm->wtFlags & TERM_VNULL)==0
3243 ){
3244 return 1;
3245 }
3246 }
3247 return 0;
3248}
3249
3250/*
3251** Structure passed to the whereIsCoveringIndex Walker callback.
3252*/
3253struct CoveringIndexCheck {
3254 Index *pIdx; /* The index */
3255 int iTabCur; /* Cursor number for the corresponding table */
3256};
3257
3258/*
3259** Information passed in is pWalk->u.pCovIdxCk. Call is pCk.
3260**
3261** If the Expr node references the table with cursor pCk->iTabCur, then
3262** make sure that column is covered by the index pCk->pIdx. We know that
3263** all columns less than 63 (really BMS-1) are covered, so we don't need
3264** to check them. But we do need to check any column at 63 or greater.
3265**
3266** If the index does not cover the column, then set pWalk->eCode to
3267** non-zero and return WRC_Abort to stop the search.
3268**
3269** If this node does not disprove that the index can be a covering index,
3270** then just return WRC_Continue, to continue the search.
3271*/
3272static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){
3273 int i; /* Loop counter */
3274 const Index *pIdx; /* The index of interest */
3275 const i16 *aiColumn; /* Columns contained in the index */
3276 u16 nColumn; /* Number of columns in the index */
3277 if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_AGG_COLUMN ) return WRC_Continue;
3278 if( pExpr->iColumn<(BMS-1) ) return WRC_Continue;
3279 if( pExpr->iTable!=pWalk->u.pCovIdxCk->iTabCur ) return WRC_Continue;
3280 pIdx = pWalk->u.pCovIdxCk->pIdx;
3281 aiColumn = pIdx->aiColumn;
3282 nColumn = pIdx->nColumn;
3283 for(i=0; i<nColumn; i++){
3284 if( aiColumn[i]==pExpr->iColumn ) return WRC_Continue;
3285 }
3286 pWalk->eCode = 1;
3287 return WRC_Abort;
3288}
3289
3290
3291/*
3292** pIdx is an index that covers all of the low-number columns used by
3293** pWInfo->pSelect (columns from 0 through 62). But there are columns
3294** in pWInfo->pSelect beyond 62. This routine tries to answer the question
3295** of whether pIdx covers *all* columns in the query.
3296**
3297** Return 0 if pIdx is a covering index. Return non-zero if pIdx is
3298** not a covering index or if we are unable to determine if pIdx is a
3299** covering index.
3300**
3301** This routine is an optimization. It is always safe to return non-zero.
3302** But returning zero when non-zero should have been returned can lead to
3303** incorrect bytecode and assertion faults.
3304*/
3305static SQLITE_NOINLINE u32 whereIsCoveringIndex(
3306 WhereInfo *pWInfo, /* The WHERE clause context */
3307 Index *pIdx, /* Index that is being tested */
3308 int iTabCur /* Cursor for the table being indexed */
3309){
3310 int i;
3311 struct CoveringIndexCheck ck;
3312 Walker w;
3313 if( pWInfo->pSelect==0 ){
3314 /* We don't have access to the full query, so we cannot check to see
3315 ** if pIdx is covering. Assume it is not. */
3316 return 1;
3317 }
3318 for(i=0; i<pIdx->nColumn; i++){
3319 if( pIdx->aiColumn[i]>=BMS-1 ) break;
3320 }
3321 if( i>=pIdx->nColumn ){
3322 /* pIdx does not index any columns greater than 62, but we know from
3323 ** colMask that columns greater than 62 are used, so this is not a
3324 ** covering index */
3325 return 1;
3326 }
3327 ck.pIdx = pIdx;
3328 ck.iTabCur = iTabCur;
3329 memset(&w, 0, sizeof(w));
3330 w.xExprCallback = whereIsCoveringIndexWalkCallback;
3331 w.xSelectCallback = sqlite3SelectWalkNoop;
3332 w.u.pCovIdxCk = &ck;
3333 w.eCode = 0;
3334 sqlite3WalkSelect(&w, pWInfo->pSelect);
3335 return w.eCode;
3336}
3337
3338/*
3339** Add all WhereLoop objects for a single table of the join where the table
3340** is identified by pBuilder->pNew->iTab. That table is guaranteed to be
3341** a b-tree table, not a virtual table.
3342**
3343** The costs (WhereLoop.rRun) of the b-tree loops added by this function
3344** are calculated as follows:
3345**
3346** For a full scan, assuming the table (or index) contains nRow rows:
3347**
3348** cost = nRow * 3.0 // full-table scan
3349** cost = nRow * K // scan of covering index
3350** cost = nRow * (K+3.0) // scan of non-covering index
3351**
3352** where K is a value between 1.1 and 3.0 set based on the relative
3353** estimated average size of the index and table records.
3354**
3355** For an index scan, where nVisit is the number of index rows visited
3356** by the scan, and nSeek is the number of seek operations required on
3357** the index b-tree:
3358**
3359** cost = nSeek * (log(nRow) + K * nVisit) // covering index
3360** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index
3361**
3362** Normally, nSeek is 1. nSeek values greater than 1 come about if the
3363** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
3364** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
3365**
3366** The estimated values (nRow, nVisit, nSeek) often contain a large amount
3367** of uncertainty. For this reason, scoring is designed to pick plans that
3368** "do the least harm" if the estimates are inaccurate. For example, a
3369** log(nRow) factor is omitted from a non-covering index scan in order to
3370** bias the scoring in favor of using an index, since the worst-case
3371** performance of using an index is far better than the worst-case performance
3372** of a full table scan.
3373*/
3374static int whereLoopAddBtree(
3375 WhereLoopBuilder *pBuilder, /* WHERE clause information */
3376 Bitmask mPrereq /* Extra prerequesites for using this table */
3377){
3378 WhereInfo *pWInfo; /* WHERE analysis context */
3379 Index *pProbe; /* An index we are evaluating */
3380 Index sPk; /* A fake index object for the primary key */
3381 LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */
3382 i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */
3383 SrcList *pTabList; /* The FROM clause */
3384 SrcItem *pSrc; /* The FROM clause btree term to add */
3385 WhereLoop *pNew; /* Template WhereLoop object */
3386 int rc = SQLITE_OK; /* Return code */
3387 int iSortIdx = 1; /* Index number */
3388 int b; /* A boolean value */
3389 LogEst rSize; /* number of rows in the table */
3390 WhereClause *pWC; /* The parsed WHERE clause */
3391 Table *pTab; /* Table being queried */
3392
3393 pNew = pBuilder->pNew;
3394 pWInfo = pBuilder->pWInfo;
3395 pTabList = pWInfo->pTabList;
3396 pSrc = pTabList->a + pNew->iTab;
3397 pTab = pSrc->pTab;
3398 pWC = pBuilder->pWC;
3399 assert( !IsVirtual(pSrc->pTab) );
3400
3401 if( pSrc->fg.isIndexedBy ){
3402 assert( pSrc->fg.isCte==0 );
3403 /* An INDEXED BY clause specifies a particular index to use */
3404 pProbe = pSrc->u2.pIBIndex;
3405 }else if( !HasRowid(pTab) ){
3406 pProbe = pTab->pIndex;
3407 }else{
3408 /* There is no INDEXED BY clause. Create a fake Index object in local
3409 ** variable sPk to represent the rowid primary key index. Make this
3410 ** fake index the first in a chain of Index objects with all of the real
3411 ** indices to follow */
3412 Index *pFirst; /* First of real indices on the table */
3413 memset(&sPk, 0, sizeof(Index));
3414 sPk.nKeyCol = 1;
3415 sPk.nColumn = 1;
3416 sPk.aiColumn = &aiColumnPk;
3417 sPk.aiRowLogEst = aiRowEstPk;
3418 sPk.onError = OE_Replace;
3419 sPk.pTable = pTab;
3420 sPk.szIdxRow = pTab->szTabRow;
3421 sPk.idxType = SQLITE_IDXTYPE_IPK;
3422 aiRowEstPk[0] = pTab->nRowLogEst;
3423 aiRowEstPk[1] = 0;
3424 pFirst = pSrc->pTab->pIndex;
3425 if( pSrc->fg.notIndexed==0 ){
3426 /* The real indices of the table are only considered if the
3427 ** NOT INDEXED qualifier is omitted from the FROM clause */
3428 sPk.pNext = pFirst;
3429 }
3430 pProbe = &sPk;
3431 }
3432 rSize = pTab->nRowLogEst;
3433
3434#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
3435 /* Automatic indexes */
3436 if( !pBuilder->pOrSet /* Not part of an OR optimization */
3437 && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0
3438 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
3439 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */
3440 && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */
3441 && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */
3442 && !pSrc->fg.isCorrelated /* Not a correlated subquery */
3443 && !pSrc->fg.isRecursive /* Not a recursive common table expression. */
3444 && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */
3445 ){
3446 /* Generate auto-index WhereLoops */
3447 LogEst rLogSize; /* Logarithm of the number of rows in the table */
3448 WhereTerm *pTerm;
3449 WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
3450 rLogSize = estLog(rSize);
3451 for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
3452 if( pTerm->prereqRight & pNew->maskSelf ) continue;
3453 if( termCanDriveIndex(pTerm, pSrc, 0) ){
3454 pNew->u.btree.nEq = 1;
3455 pNew->nSkip = 0;
3456 pNew->u.btree.pIndex = 0;
3457 pNew->nLTerm = 1;
3458 pNew->aLTerm[0] = pTerm;
3459 /* TUNING: One-time cost for computing the automatic index is
3460 ** estimated to be X*N*log2(N) where N is the number of rows in
3461 ** the table being indexed and where X is 7 (LogEst=28) for normal
3462 ** tables or 0.5 (LogEst=-10) for views and subqueries. The value
3463 ** of X is smaller for views and subqueries so that the query planner
3464 ** will be more aggressive about generating automatic indexes for
3465 ** those objects, since there is no opportunity to add schema
3466 ** indexes on subqueries and views. */
3467 pNew->rSetup = rLogSize + rSize;
3468 if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){
3469 pNew->rSetup += 28;
3470 }else{
3471 pNew->rSetup -= 10;
3472 }
3473 ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
3474 if( pNew->rSetup<0 ) pNew->rSetup = 0;
3475 /* TUNING: Each index lookup yields 20 rows in the table. This
3476 ** is more than the usual guess of 10 rows, since we have no way
3477 ** of knowing how selective the index will ultimately be. It would
3478 ** not be unreasonable to make this value much larger. */
3479 pNew->nOut = 43; assert( 43==sqlite3LogEst(20) );
3480 pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut);
3481 pNew->wsFlags = WHERE_AUTO_INDEX;
3482 pNew->prereq = mPrereq | pTerm->prereqRight;
3483 rc = whereLoopInsert(pBuilder, pNew);
3484 }
3485 }
3486 }
3487#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
3488
3489 /* Loop over all indices. If there was an INDEXED BY clause, then only
3490 ** consider index pProbe. */
3491 for(; rc==SQLITE_OK && pProbe;
3492 pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++
3493 ){
3494 if( pProbe->pPartIdxWhere!=0
3495 && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC,
3496 pProbe->pPartIdxWhere)
3497 ){
3498 testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */
3499 continue; /* Partial index inappropriate for this query */
3500 }
3501 if( pProbe->bNoQuery ) continue;
3502 rSize = pProbe->aiRowLogEst[0];
3503 pNew->u.btree.nEq = 0;
3504 pNew->u.btree.nBtm = 0;
3505 pNew->u.btree.nTop = 0;
3506 pNew->nSkip = 0;
3507 pNew->nLTerm = 0;
3508 pNew->iSortIdx = 0;
3509 pNew->rSetup = 0;
3510 pNew->prereq = mPrereq;
3511 pNew->nOut = rSize;
3512 pNew->u.btree.pIndex = pProbe;
3513 b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
3514
3515 /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
3516 assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 );
3517 if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
3518 /* Integer primary key index */
3519 pNew->wsFlags = WHERE_IPK;
3520
3521 /* Full table scan */
3522 pNew->iSortIdx = b ? iSortIdx : 0;
3523 /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an
3524 ** extra cost designed to discourage the use of full table scans,
3525 ** since index lookups have better worst-case performance if our
3526 ** stat guesses are wrong. Reduce the 3.0 penalty slightly
3527 ** (to 2.75) if we have valid STAT4 information for the table.
3528 ** At 2.75, a full table scan is preferred over using an index on
3529 ** a column with just two distinct values where each value has about
3530 ** an equal number of appearances. Without STAT4 data, we still want
3531 ** to use an index in that case, since the constraint might be for
3532 ** the scarcer of the two values, and in that case an index lookup is
3533 ** better.
3534 */
3535#ifdef SQLITE_ENABLE_STAT4
3536 pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0);
3537#else
3538 pNew->rRun = rSize + 16;
3539#endif
3540 if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){
3541 pNew->wsFlags |= WHERE_VIEWSCAN;
3542 }
3543 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
3544 whereLoopOutputAdjust(pWC, pNew, rSize);
3545 rc = whereLoopInsert(pBuilder, pNew);
3546 pNew->nOut = rSize;
3547 if( rc ) break;
3548 }else{
3549 Bitmask m;
3550 if( pProbe->isCovering ){
3551 pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
3552 m = 0;
3553 }else{
3554 m = pSrc->colUsed & pProbe->colNotIdxed;
3555 if( m==TOPBIT ){
3556 m = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
3557 }
3558 pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED;
3559 }
3560
3561 /* Full scan via index */
3562 if( b
3563 || !HasRowid(pTab)
3564 || pProbe->pPartIdxWhere!=0
3565 || pSrc->fg.isIndexedBy
3566 || ( m==0
3567 && pProbe->bUnordered==0
3568 && (pProbe->szIdxRow<pTab->szTabRow)
3569 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0
3570 && sqlite3GlobalConfig.bUseCis
3571 && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan)
3572 )
3573 ){
3574 pNew->iSortIdx = b ? iSortIdx : 0;
3575
3576 /* The cost of visiting the index rows is N*K, where K is
3577 ** between 1.1 and 3.0, depending on the relative sizes of the
3578 ** index and table rows. */
3579 pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
3580 if( m!=0 ){
3581 /* If this is a non-covering index scan, add in the cost of
3582 ** doing table lookups. The cost will be 3x the number of
3583 ** lookups. Take into account WHERE clause terms that can be
3584 ** satisfied using just the index, and that do not require a
3585 ** table lookup. */
3586 LogEst nLookup = rSize + 16; /* Base cost: N*3 */
3587 int ii;
3588 int iCur = pSrc->iCursor;
3589 WhereClause *pWC2 = &pWInfo->sWC;
3590 for(ii=0; ii<pWC2->nTerm; ii++){
3591 WhereTerm *pTerm = &pWC2->a[ii];
3592 if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
3593 break;
3594 }
3595 /* pTerm can be evaluated using just the index. So reduce
3596 ** the expected number of table lookups accordingly */
3597 if( pTerm->truthProb<=0 ){
3598 nLookup += pTerm->truthProb;
3599 }else{
3600 nLookup--;
3601 if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
3602 }
3603 }
3604
3605 pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
3606 }
3607 ApplyCostMultiplier(pNew->rRun, pTab->costMult);
3608 whereLoopOutputAdjust(pWC, pNew, rSize);
3609 if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){
3610 /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN
3611 ** because the cursor used to access the index might not be
3612 ** positioned to the correct row during the right-join no-match
3613 ** loop. */
3614 }else{
3615 rc = whereLoopInsert(pBuilder, pNew);
3616 }
3617 pNew->nOut = rSize;
3618 if( rc ) break;
3619 }
3620 }
3621
3622 pBuilder->bldFlags1 = 0;
3623 rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
3624 if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){
3625 /* If a non-unique index is used, or if a prefix of the key for
3626 ** unique index is used (making the index functionally non-unique)
3627 ** then the sqlite_stat1 data becomes important for scoring the
3628 ** plan */
3629 pTab->tabFlags |= TF_StatsUsed;
3630 }
3631#ifdef SQLITE_ENABLE_STAT4
3632 sqlite3Stat4ProbeFree(pBuilder->pRec);
3633 pBuilder->nRecValid = 0;
3634 pBuilder->pRec = 0;
3635#endif
3636 }
3637 return rc;
3638}
3639
3640#ifndef SQLITE_OMIT_VIRTUALTABLE
3641
3642/*
3643** Return true if pTerm is a virtual table LIMIT or OFFSET term.
3644*/
3645static int isLimitTerm(WhereTerm *pTerm){
3646 assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 );
3647 return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT
3648 && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET;
3649}
3650
3651/*
3652** Argument pIdxInfo is already populated with all constraints that may
3653** be used by the virtual table identified by pBuilder->pNew->iTab. This
3654** function marks a subset of those constraints usable, invokes the
3655** xBestIndex method and adds the returned plan to pBuilder.
3656**
3657** A constraint is marked usable if:
3658**
3659** * Argument mUsable indicates that its prerequisites are available, and
3660**
3661** * It is not one of the operators specified in the mExclude mask passed
3662** as the fourth argument (which in practice is either WO_IN or 0).
3663**
3664** Argument mPrereq is a mask of tables that must be scanned before the
3665** virtual table in question. These are added to the plans prerequisites
3666** before it is added to pBuilder.
3667**
3668** Output parameter *pbIn is set to true if the plan added to pBuilder
3669** uses one or more WO_IN terms, or false otherwise.
3670*/
3671static int whereLoopAddVirtualOne(
3672 WhereLoopBuilder *pBuilder,
3673 Bitmask mPrereq, /* Mask of tables that must be used. */
3674 Bitmask mUsable, /* Mask of usable tables */
3675 u16 mExclude, /* Exclude terms using these operators */
3676 sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */
3677 u16 mNoOmit, /* Do not omit these constraints */
3678 int *pbIn, /* OUT: True if plan uses an IN(...) op */
3679 int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */
3680){
3681 WhereClause *pWC = pBuilder->pWC;
3682 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
3683 struct sqlite3_index_constraint *pIdxCons;
3684 struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage;
3685 int i;
3686 int mxTerm;
3687 int rc = SQLITE_OK;
3688 WhereLoop *pNew = pBuilder->pNew;
3689 Parse *pParse = pBuilder->pWInfo->pParse;
3690 SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab];
3691 int nConstraint = pIdxInfo->nConstraint;
3692
3693 assert( (mUsable & mPrereq)==mPrereq );
3694 *pbIn = 0;
3695 pNew->prereq = mPrereq;
3696
3697 /* Set the usable flag on the subset of constraints identified by
3698 ** arguments mUsable and mExclude. */
3699 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
3700 for(i=0; i<nConstraint; i++, pIdxCons++){
3701 WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset];
3702 pIdxCons->usable = 0;
3703 if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight
3704 && (pTerm->eOperator & mExclude)==0
3705 && (pbRetryLimit || !isLimitTerm(pTerm))
3706 ){
3707 pIdxCons->usable = 1;
3708 }
3709 }
3710
3711 /* Initialize the output fields of the sqlite3_index_info structure */
3712 memset(pUsage, 0, sizeof(pUsage[0])*nConstraint);
3713 assert( pIdxInfo->needToFreeIdxStr==0 );
3714 pIdxInfo->idxStr = 0;
3715 pIdxInfo->idxNum = 0;
3716 pIdxInfo->orderByConsumed = 0;
3717 pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
3718 pIdxInfo->estimatedRows = 25;
3719 pIdxInfo->idxFlags = 0;
3720 pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
3721 pHidden->mHandleIn = 0;
3722
3723 /* Invoke the virtual table xBestIndex() method */
3724 rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo);
3725 if( rc ){
3726 if( rc==SQLITE_CONSTRAINT ){
3727 /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means
3728 ** that the particular combination of parameters provided is unusable.
3729 ** Make no entries in the loop table.
3730 */
3731 WHERETRACE(0xffff, (" ^^^^--- non-viable plan rejected!\n"));
3732 return SQLITE_OK;
3733 }
3734 return rc;
3735 }
3736
3737 mxTerm = -1;
3738 assert( pNew->nLSlot>=nConstraint );
3739 memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint );
3740 memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab));
3741 pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
3742 for(i=0; i<nConstraint; i++, pIdxCons++){
3743 int iTerm;
3744 if( (iTerm = pUsage[i].argvIndex - 1)>=0 ){
3745 WhereTerm *pTerm;
3746 int j = pIdxCons->iTermOffset;
3747 if( iTerm>=nConstraint
3748 || j<0
3749 || j>=pWC->nTerm
3750 || pNew->aLTerm[iTerm]!=0
3751 || pIdxCons->usable==0
3752 ){
3753 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
3754 testcase( pIdxInfo->needToFreeIdxStr );
3755 return SQLITE_ERROR;
3756 }
3757 testcase( iTerm==nConstraint-1 );
3758 testcase( j==0 );
3759 testcase( j==pWC->nTerm-1 );
3760 pTerm = &pWC->a[j];
3761 pNew->prereq |= pTerm->prereqRight;
3762 assert( iTerm<pNew->nLSlot );
3763 pNew->aLTerm[iTerm] = pTerm;
3764 if( iTerm>mxTerm ) mxTerm = iTerm;
3765 testcase( iTerm==15 );
3766 testcase( iTerm==16 );
3767 if( pUsage[i].omit ){
3768 if( i<16 && ((1<<i)&mNoOmit)==0 ){
3769 testcase( i!=iTerm );
3770 pNew->u.vtab.omitMask |= 1<<iTerm;
3771 }else{
3772 testcase( i!=iTerm );
3773 }
3774 if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){
3775 pNew->u.vtab.bOmitOffset = 1;
3776 }
3777 }
3778 if( SMASKBIT32(i) & pHidden->mHandleIn ){
3779 pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm);
3780 }else if( (pTerm->eOperator & WO_IN)!=0 ){
3781 /* A virtual table that is constrained by an IN clause may not
3782 ** consume the ORDER BY clause because (1) the order of IN terms
3783 ** is not necessarily related to the order of output terms and
3784 ** (2) Multiple outputs from a single IN value will not merge
3785 ** together. */
3786 pIdxInfo->orderByConsumed = 0;
3787 pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
3788 *pbIn = 1; assert( (mExclude & WO_IN)==0 );
3789 }
3790
3791 assert( pbRetryLimit || !isLimitTerm(pTerm) );
3792 if( isLimitTerm(pTerm) && *pbIn ){
3793 /* If there is an IN(...) term handled as an == (separate call to
3794 ** xFilter for each value on the RHS of the IN) and a LIMIT or
3795 ** OFFSET term handled as well, the plan is unusable. Set output
3796 ** variable *pbRetryLimit to true to tell the caller to retry with
3797 ** LIMIT and OFFSET disabled. */
3798 if( pIdxInfo->needToFreeIdxStr ){
3799 sqlite3_free(pIdxInfo->idxStr);
3800 pIdxInfo->idxStr = 0;
3801 pIdxInfo->needToFreeIdxStr = 0;
3802 }
3803 *pbRetryLimit = 1;
3804 return SQLITE_OK;
3805 }
3806 }
3807 }
3808
3809 pNew->nLTerm = mxTerm+1;
3810 for(i=0; i<=mxTerm; i++){
3811 if( pNew->aLTerm[i]==0 ){
3812 /* The non-zero argvIdx values must be contiguous. Raise an
3813 ** error if they are not */
3814 sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
3815 testcase( pIdxInfo->needToFreeIdxStr );
3816 return SQLITE_ERROR;
3817 }
3818 }
3819 assert( pNew->nLTerm<=pNew->nLSlot );
3820 pNew->u.vtab.idxNum = pIdxInfo->idxNum;
3821 pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
3822 pIdxInfo->needToFreeIdxStr = 0;
3823 pNew->u.vtab.idxStr = pIdxInfo->idxStr;
3824 pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
3825 pIdxInfo->nOrderBy : 0);
3826 pNew->rSetup = 0;
3827 pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
3828 pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
3829
3830 /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
3831 ** that the scan will visit at most one row. Clear it otherwise. */
3832 if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
3833 pNew->wsFlags |= WHERE_ONEROW;
3834 }else{
3835 pNew->wsFlags &= ~WHERE_ONEROW;
3836 }
3837 rc = whereLoopInsert(pBuilder, pNew);
3838 if( pNew->u.vtab.needFree ){
3839 sqlite3_free(pNew->u.vtab.idxStr);
3840 pNew->u.vtab.needFree = 0;
3841 }
3842 WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
3843 *pbIn, (sqlite3_uint64)mPrereq,
3844 (sqlite3_uint64)(pNew->prereq & ~mPrereq)));
3845
3846 return rc;
3847}
3848
3849/*
3850** Return the collating sequence for a constraint passed into xBestIndex.
3851**
3852** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex.
3853** This routine depends on there being a HiddenIndexInfo structure immediately
3854** following the sqlite3_index_info structure.
3855**
3856** Return a pointer to the collation name:
3857**
3858** 1. If there is an explicit COLLATE operator on the constaint, return it.
3859**
3860** 2. Else, if the column has an alternative collation, return that.
3861**
3862** 3. Otherwise, return "BINARY".
3863*/
3864const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
3865 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
3866 const char *zRet = 0;
3867 if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
3868 CollSeq *pC = 0;
3869 int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
3870 Expr *pX = pHidden->pWC->a[iTerm].pExpr;
3871 if( pX->pLeft ){
3872 pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX);
3873 }
3874 zRet = (pC ? pC->zName : sqlite3StrBINARY);
3875 }
3876 return zRet;
3877}
3878
3879/*
3880** Return true if constraint iCons is really an IN(...) constraint, or
3881** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0)
3882** or clear (if bHandle==0) the flag to handle it using an iterator.
3883*/
3884int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){
3885 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
3886 u32 m = SMASKBIT32(iCons);
3887 if( m & pHidden->mIn ){
3888 if( bHandle==0 ){
3889 pHidden->mHandleIn &= ~m;
3890 }else if( bHandle>0 ){
3891 pHidden->mHandleIn |= m;
3892 }
3893 return 1;
3894 }
3895 return 0;
3896}
3897
3898/*
3899** This interface is callable from within the xBestIndex callback only.
3900**
3901** If possible, set (*ppVal) to point to an object containing the value
3902** on the right-hand-side of constraint iCons.
3903*/
3904int sqlite3_vtab_rhs_value(
3905 sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */
3906 int iCons, /* Constraint for which RHS is wanted */
3907 sqlite3_value **ppVal /* Write value extracted here */
3908){
3909 HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1];
3910 sqlite3_value *pVal = 0;
3911 int rc = SQLITE_OK;
3912 if( iCons<0 || iCons>=pIdxInfo->nConstraint ){
3913 rc = SQLITE_MISUSE; /* EV: R-30545-25046 */
3914 }else{
3915 if( pH->aRhs[iCons]==0 ){
3916 WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset];
3917 rc = sqlite3ValueFromExpr(
3918 pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db),
3919 SQLITE_AFF_BLOB, &pH->aRhs[iCons]
3920 );
3921 testcase( rc!=SQLITE_OK );
3922 }
3923 pVal = pH->aRhs[iCons];
3924 }
3925 *ppVal = pVal;
3926
3927 if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */
3928 rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */
3929 }
3930
3931 return rc;
3932}
3933
3934/*
3935** Return true if ORDER BY clause may be handled as DISTINCT.
3936*/
3937int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
3938 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
3939 assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 );
3940 return pHidden->eDistinct;
3941}
3942
3943#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \
3944 && !defined(SQLITE_OMIT_VIRTUALTABLE)
3945/*
3946** Cause the prepared statement that is associated with a call to
3947** xBestIndex to potentiall use all schemas. If the statement being
3948** prepared is read-only, then just start read transactions on all
3949** schemas. But if this is a write operation, start writes on all
3950** schemas.
3951**
3952** This is used by the (built-in) sqlite_dbpage virtual table.
3953*/
3954void sqlite3VtabUsesAllSchemas(sqlite3_index_info *pIdxInfo){
3955 HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
3956 Parse *pParse = pHidden->pParse;
3957 int nDb = pParse->db->nDb;
3958 int i;
3959 for(i=0; i<nDb; i++){
3960 sqlite3CodeVerifySchema(pParse, i);
3961 }
3962 if( pParse->writeMask ){
3963 for(i=0; i<nDb; i++){
3964 sqlite3BeginWriteOperation(pParse, 0, i);
3965 }
3966 }
3967}
3968#endif
3969
3970/*
3971** Add all WhereLoop objects for a table of the join identified by
3972** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table.
3973**
3974** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
3975** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
3976** entries that occur before the virtual table in the FROM clause and are
3977** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
3978** mUnusable mask contains all FROM clause entries that occur after the
3979** virtual table and are separated from it by at least one LEFT or
3980** CROSS JOIN.
3981**
3982** For example, if the query were:
3983**
3984** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
3985**
3986** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
3987**
3988** All the tables in mPrereq must be scanned before the current virtual
3989** table. So any terms for which all prerequisites are satisfied by
3990** mPrereq may be specified as "usable" in all calls to xBestIndex.
3991** Conversely, all tables in mUnusable must be scanned after the current
3992** virtual table, so any terms for which the prerequisites overlap with
3993** mUnusable should always be configured as "not-usable" for xBestIndex.
3994*/
3995static int whereLoopAddVirtual(
3996 WhereLoopBuilder *pBuilder, /* WHERE clause information */
3997 Bitmask mPrereq, /* Tables that must be scanned before this one */
3998 Bitmask mUnusable /* Tables that must be scanned after this one */
3999){
4000 int rc = SQLITE_OK; /* Return code */
4001 WhereInfo *pWInfo; /* WHERE analysis context */
4002 Parse *pParse; /* The parsing context */
4003 WhereClause *pWC; /* The WHERE clause */
4004 SrcItem *pSrc; /* The FROM clause term to search */
4005 sqlite3_index_info *p; /* Object to pass to xBestIndex() */
4006 int nConstraint; /* Number of constraints in p */
4007 int bIn; /* True if plan uses IN(...) operator */
4008 WhereLoop *pNew;
4009 Bitmask mBest; /* Tables used by best possible plan */
4010 u16 mNoOmit;
4011 int bRetry = 0; /* True to retry with LIMIT/OFFSET disabled */
4012
4013 assert( (mPrereq & mUnusable)==0 );
4014 pWInfo = pBuilder->pWInfo;
4015 pParse = pWInfo->pParse;
4016 pWC = pBuilder->pWC;
4017 pNew = pBuilder->pNew;
4018 pSrc = &pWInfo->pTabList->a[pNew->iTab];
4019 assert( IsVirtual(pSrc->pTab) );
4020 p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit);
4021 if( p==0 ) return SQLITE_NOMEM_BKPT;
4022 pNew->rSetup = 0;
4023 pNew->wsFlags = WHERE_VIRTUALTABLE;
4024 pNew->nLTerm = 0;
4025 pNew->u.vtab.needFree = 0;
4026 nConstraint = p->nConstraint;
4027 if( whereLoopResize(pParse->db, pNew, nConstraint) ){
4028 freeIndexInfo(pParse->db, p);
4029 return SQLITE_NOMEM_BKPT;
4030 }
4031
4032 /* First call xBestIndex() with all constraints usable. */
4033 WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName));
4034 WHERETRACE(0x40, (" VirtualOne: all usable\n"));
4035 rc = whereLoopAddVirtualOne(
4036 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry
4037 );
4038 if( bRetry ){
4039 assert( rc==SQLITE_OK );
4040 rc = whereLoopAddVirtualOne(
4041 pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0
4042 );
4043 }
4044
4045 /* If the call to xBestIndex() with all terms enabled produced a plan
4046 ** that does not require any source tables (IOW: a plan with mBest==0)
4047 ** and does not use an IN(...) operator, then there is no point in making
4048 ** any further calls to xBestIndex() since they will all return the same
4049 ** result (if the xBestIndex() implementation is sane). */
4050 if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){
4051 int seenZero = 0; /* True if a plan with no prereqs seen */
4052 int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */
4053 Bitmask mPrev = 0;
4054 Bitmask mBestNoIn = 0;
4055
4056 /* If the plan produced by the earlier call uses an IN(...) term, call
4057 ** xBestIndex again, this time with IN(...) terms disabled. */
4058 if( bIn ){
4059 WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n"));
4060 rc = whereLoopAddVirtualOne(
4061 pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0);
4062 assert( bIn==0 );
4063 mBestNoIn = pNew->prereq & ~mPrereq;
4064 if( mBestNoIn==0 ){
4065 seenZero = 1;
4066 seenZeroNoIN = 1;
4067 }
4068 }
4069
4070 /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
4071 ** in the set of terms that apply to the current virtual table. */
4072 while( rc==SQLITE_OK ){
4073 int i;
4074 Bitmask mNext = ALLBITS;
4075 assert( mNext>0 );
4076 for(i=0; i<nConstraint; i++){
4077 Bitmask mThis = (
4078 pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq
4079 );
4080 if( mThis>mPrev && mThis<mNext ) mNext = mThis;
4081 }
4082 mPrev = mNext;
4083 if( mNext==ALLBITS ) break;
4084 if( mNext==mBest || mNext==mBestNoIn ) continue;
4085 WHERETRACE(0x40, (" VirtualOne: mPrev=%04llx mNext=%04llx\n",
4086 (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext));
4087 rc = whereLoopAddVirtualOne(
4088 pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn, 0);
4089 if( pNew->prereq==mPrereq ){
4090 seenZero = 1;
4091 if( bIn==0 ) seenZeroNoIN = 1;
4092 }
4093 }
4094
4095 /* If the calls to xBestIndex() in the above loop did not find a plan
4096 ** that requires no source tables at all (i.e. one guaranteed to be
4097 ** usable), make a call here with all source tables disabled */
4098 if( rc==SQLITE_OK && seenZero==0 ){
4099 WHERETRACE(0x40, (" VirtualOne: all disabled\n"));
4100 rc = whereLoopAddVirtualOne(
4101 pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0);
4102 if( bIn==0 ) seenZeroNoIN = 1;
4103 }
4104
4105 /* If the calls to xBestIndex() have so far failed to find a plan
4106 ** that requires no source tables at all and does not use an IN(...)
4107 ** operator, make a final call to obtain one here. */
4108 if( rc==SQLITE_OK && seenZeroNoIN==0 ){
4109 WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n"));
4110 rc = whereLoopAddVirtualOne(
4111 pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0);
4112 }
4113 }
4114
4115 if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
4116 freeIndexInfo(pParse->db, p);
4117 WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc));
4118 return rc;
4119}
4120#endif /* SQLITE_OMIT_VIRTUALTABLE */
4121
4122/*
4123** Add WhereLoop entries to handle OR terms. This works for either
4124** btrees or virtual tables.
4125*/
4126static int whereLoopAddOr(
4127 WhereLoopBuilder *pBuilder,
4128 Bitmask mPrereq,
4129 Bitmask mUnusable
4130){
4131 WhereInfo *pWInfo = pBuilder->pWInfo;
4132 WhereClause *pWC;
4133 WhereLoop *pNew;
4134 WhereTerm *pTerm, *pWCEnd;
4135 int rc = SQLITE_OK;
4136 int iCur;
4137 WhereClause tempWC;
4138 WhereLoopBuilder sSubBuild;
4139 WhereOrSet sSum, sCur;
4140 SrcItem *pItem;
4141
4142 pWC = pBuilder->pWC;
4143 pWCEnd = pWC->a + pWC->nTerm;
4144 pNew = pBuilder->pNew;
4145 memset(&sSum, 0, sizeof(sSum));
4146 pItem = pWInfo->pTabList->a + pNew->iTab;
4147 iCur = pItem->iCursor;
4148
4149 /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */
4150 if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK;
4151
4152 for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
4153 if( (pTerm->eOperator & WO_OR)!=0
4154 && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0
4155 ){
4156 WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
4157 WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
4158 WhereTerm *pOrTerm;
4159 int once = 1;
4160 int i, j;
4161
4162 sSubBuild = *pBuilder;
4163 sSubBuild.pOrSet = &sCur;
4164
4165 WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm));
4166 for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
4167 if( (pOrTerm->eOperator & WO_AND)!=0 ){
4168 sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
4169 }else if( pOrTerm->leftCursor==iCur ){
4170 tempWC.pWInfo = pWC->pWInfo;
4171 tempWC.pOuter = pWC;
4172 tempWC.op = TK_AND;
4173 tempWC.nTerm = 1;
4174 tempWC.nBase = 1;
4175 tempWC.a = pOrTerm;
4176 sSubBuild.pWC = &tempWC;
4177 }else{
4178 continue;
4179 }
4180 sCur.n = 0;
4181#ifdef WHERETRACE_ENABLED
4182 WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n",
4183 (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
4184 if( sqlite3WhereTrace & 0x400 ){
4185 sqlite3WhereClausePrint(sSubBuild.pWC);
4186 }
4187#endif
4188#ifndef SQLITE_OMIT_VIRTUALTABLE
4189 if( IsVirtual(pItem->pTab) ){
4190 rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
4191 }else
4192#endif
4193 {
4194 rc = whereLoopAddBtree(&sSubBuild, mPrereq);
4195 }
4196 if( rc==SQLITE_OK ){
4197 rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable);
4198 }
4199 assert( rc==SQLITE_OK || rc==SQLITE_DONE || sCur.n==0
4200 || rc==SQLITE_NOMEM );
4201 testcase( rc==SQLITE_NOMEM && sCur.n>0 );
4202 testcase( rc==SQLITE_DONE );
4203 if( sCur.n==0 ){
4204 sSum.n = 0;
4205 break;
4206 }else if( once ){
4207 whereOrMove(&sSum, &sCur);
4208 once = 0;
4209 }else{
4210 WhereOrSet sPrev;
4211 whereOrMove(&sPrev, &sSum);
4212 sSum.n = 0;
4213 for(i=0; i<sPrev.n; i++){
4214 for(j=0; j<sCur.n; j++){
4215 whereOrInsert(&sSum, sPrev.a[i].prereq | sCur.a[j].prereq,
4216 sqlite3LogEstAdd(sPrev.a[i].rRun, sCur.a[j].rRun),
4217 sqlite3LogEstAdd(sPrev.a[i].nOut, sCur.a[j].nOut));
4218 }
4219 }
4220 }
4221 }
4222 pNew->nLTerm = 1;
4223 pNew->aLTerm[0] = pTerm;
4224 pNew->wsFlags = WHERE_MULTI_OR;
4225 pNew->rSetup = 0;
4226 pNew->iSortIdx = 0;
4227 memset(&pNew->u, 0, sizeof(pNew->u));
4228 for(i=0; rc==SQLITE_OK && i<sSum.n; i++){
4229 /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
4230 ** of all sub-scans required by the OR-scan. However, due to rounding
4231 ** errors, it may be that the cost of the OR-scan is equal to its
4232 ** most expensive sub-scan. Add the smallest possible penalty
4233 ** (equivalent to multiplying the cost by 1.07) to ensure that
4234 ** this does not happen. Otherwise, for WHERE clauses such as the
4235 ** following where there is an index on "y":
4236 **
4237 ** WHERE likelihood(x=?, 0.99) OR y=?
4238 **
4239 ** the planner may elect to "OR" together a full-table scan and an
4240 ** index lookup. And other similarly odd results. */
4241 pNew->rRun = sSum.a[i].rRun + 1;
4242 pNew->nOut = sSum.a[i].nOut;
4243 pNew->prereq = sSum.a[i].prereq;
4244 rc = whereLoopInsert(pBuilder, pNew);
4245 }
4246 WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm));
4247 }
4248 }
4249 return rc;
4250}
4251
4252/*
4253** Add all WhereLoop objects for all tables
4254*/
4255static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
4256 WhereInfo *pWInfo = pBuilder->pWInfo;
4257 Bitmask mPrereq = 0;
4258 Bitmask mPrior = 0;
4259 int iTab;
4260 SrcList *pTabList = pWInfo->pTabList;
4261 SrcItem *pItem;
4262 SrcItem *pEnd = &pTabList->a[pWInfo->nLevel];
4263 sqlite3 *db = pWInfo->pParse->db;
4264 int rc = SQLITE_OK;
4265 int bFirstPastRJ = 0;
4266 int hasRightJoin = 0;
4267 WhereLoop *pNew;
4268
4269
4270 /* Loop over the tables in the join, from left to right */
4271 pNew = pBuilder->pNew;
4272
4273 /* Verify that pNew has already been initialized */
4274 assert( pNew->nLTerm==0 );
4275 assert( pNew->wsFlags==0 );
4276 assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) );
4277 assert( pNew->aLTerm!=0 );
4278
4279 pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT;
4280 for(iTab=0, pItem=pTabList->a; pItem<pEnd; iTab++, pItem++){
4281 Bitmask mUnusable = 0;
4282 pNew->iTab = iTab;
4283 pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR;
4284 pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor);
4285 if( bFirstPastRJ
4286 || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0
4287 ){
4288 /* Add prerequisites to prevent reordering of FROM clause terms
4289 ** across CROSS joins and outer joins. The bFirstPastRJ boolean
4290 ** prevents the right operand of a RIGHT JOIN from being swapped with
4291 ** other elements even further to the right.
4292 **
4293 ** The JT_LTORJ case and the hasRightJoin flag work together to
4294 ** prevent FROM-clause terms from moving from the right side of
4295 ** a LEFT JOIN over to the left side of that join if the LEFT JOIN
4296 ** is itself on the left side of a RIGHT JOIN.
4297 */
4298 if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1;
4299 mPrereq |= mPrior;
4300 bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0;
4301 }else if( !hasRightJoin ){
4302 mPrereq = 0;
4303 }
4304#ifndef SQLITE_OMIT_VIRTUALTABLE
4305 if( IsVirtual(pItem->pTab) ){
4306 SrcItem *p;
4307 for(p=&pItem[1]; p<pEnd; p++){
4308 if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){
4309 mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor);
4310 }
4311 }
4312 rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
4313 }else
4314#endif /* SQLITE_OMIT_VIRTUALTABLE */
4315 {
4316 rc = whereLoopAddBtree(pBuilder, mPrereq);
4317 }
4318 if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){
4319 rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable);
4320 }
4321 mPrior |= pNew->maskSelf;
4322 if( rc || db->mallocFailed ){
4323 if( rc==SQLITE_DONE ){
4324 /* We hit the query planner search limit set by iPlanLimit */
4325 sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search");
4326 rc = SQLITE_OK;
4327 }else{
4328 break;
4329 }
4330 }
4331 }
4332
4333 whereLoopClear(db, pNew);
4334 return rc;
4335}
4336
4337/*
4338** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
4339** parameters) to see if it outputs rows in the requested ORDER BY
4340** (or GROUP BY) without requiring a separate sort operation. Return N:
4341**
4342** N>0: N terms of the ORDER BY clause are satisfied
4343** N==0: No terms of the ORDER BY clause are satisfied
4344** N<0: Unknown yet how many terms of ORDER BY might be satisfied.
4345**
4346** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
4347** strict. With GROUP BY and DISTINCT the only requirement is that
4348** equivalent rows appear immediately adjacent to one another. GROUP BY
4349** and DISTINCT do not require rows to appear in any particular order as long
4350** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT
4351** the pOrderBy terms can be matched in any order. With ORDER BY, the
4352** pOrderBy terms must be matched in strict left-to-right order.
4353*/
4354static i8 wherePathSatisfiesOrderBy(
4355 WhereInfo *pWInfo, /* The WHERE clause */
4356 ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */
4357 WherePath *pPath, /* The WherePath to check */
4358 u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
4359 u16 nLoop, /* Number of entries in pPath->aLoop[] */
4360 WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */
4361 Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */
4362){
4363 u8 revSet; /* True if rev is known */
4364 u8 rev; /* Composite sort order */
4365 u8 revIdx; /* Index sort order */
4366 u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */
4367 u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */
4368 u8 isMatch; /* iColumn matches a term of the ORDER BY clause */
4369 u16 eqOpMask; /* Allowed equality operators */
4370 u16 nKeyCol; /* Number of key columns in pIndex */
4371 u16 nColumn; /* Total number of ordered columns in the index */
4372 u16 nOrderBy; /* Number terms in the ORDER BY clause */
4373 int iLoop; /* Index of WhereLoop in pPath being processed */
4374 int i, j; /* Loop counters */
4375 int iCur; /* Cursor number for current WhereLoop */
4376 int iColumn; /* A column number within table iCur */
4377 WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */
4378 WhereTerm *pTerm; /* A single term of the WHERE clause */
4379 Expr *pOBExpr; /* An expression from the ORDER BY clause */
4380 CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */
4381 Index *pIndex; /* The index associated with pLoop */
4382 sqlite3 *db = pWInfo->pParse->db; /* Database connection */
4383 Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */
4384 Bitmask obDone; /* Mask of all ORDER BY terms */
4385 Bitmask orderDistinctMask; /* Mask of all well-ordered loops */
4386 Bitmask ready; /* Mask of inner loops */
4387
4388 /*
4389 ** We say the WhereLoop is "one-row" if it generates no more than one
4390 ** row of output. A WhereLoop is one-row if all of the following are true:
4391 ** (a) All index columns match with WHERE_COLUMN_EQ.
4392 ** (b) The index is unique
4393 ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row.
4394 ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags.
4395 **
4396 ** We say the WhereLoop is "order-distinct" if the set of columns from
4397 ** that WhereLoop that are in the ORDER BY clause are different for every
4398 ** row of the WhereLoop. Every one-row WhereLoop is automatically
4399 ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause
4400 ** is not order-distinct. To be order-distinct is not quite the same as being
4401 ** UNIQUE since a UNIQUE column or index can have multiple rows that
4402 ** are NULL and NULL values are equivalent for the purpose of order-distinct.
4403 ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
4404 **
4405 ** The rowid for a table is always UNIQUE and NOT NULL so whenever the
4406 ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is
4407 ** automatically order-distinct.
4408 */
4409
4410 assert( pOrderBy!=0 );
4411 if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0;
4412
4413 nOrderBy = pOrderBy->nExpr;
4414 testcase( nOrderBy==BMS-1 );
4415 if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */
4416 isOrderDistinct = 1;
4417 obDone = MASKBIT(nOrderBy)-1;
4418 orderDistinctMask = 0;
4419 ready = 0;
4420 eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
4421 if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){
4422 eqOpMask |= WO_IN;
4423 }
4424 for(iLoop=0; isOrderDistinct && obSat<obDone && iLoop<=nLoop; iLoop++){
4425 if( iLoop>0 ) ready |= pLoop->maskSelf;
4426 if( iLoop<nLoop ){
4427 pLoop = pPath->aLoop[iLoop];
4428 if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
4429 }else{
4430 pLoop = pLast;
4431 }
4432 if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
4433 if( pLoop->u.vtab.isOrdered
4434 && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY)
4435 ){
4436 obSat = obDone;
4437 }
4438 break;
4439 }else if( wctrlFlags & WHERE_DISTINCTBY ){
4440 pLoop->u.btree.nDistinctCol = 0;
4441 }
4442 iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
4443
4444 /* Mark off any ORDER BY term X that is a column in the table of
4445 ** the current loop for which there is term in the WHERE
4446 ** clause of the form X IS NULL or X=? that reference only outer
4447 ** loops.
4448 */
4449 for(i=0; i<nOrderBy; i++){
4450 if( MASKBIT(i) & obSat ) continue;
4451 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
4452 if( NEVER(pOBExpr==0) ) continue;
4453 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
4454 if( pOBExpr->iTable!=iCur ) continue;
4455 pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
4456 ~ready, eqOpMask, 0);
4457 if( pTerm==0 ) continue;
4458 if( pTerm->eOperator==WO_IN ){
4459 /* IN terms are only valid for sorting in the ORDER BY LIMIT
4460 ** optimization, and then only if they are actually used
4461 ** by the query plan */
4462 assert( wctrlFlags &
4463 (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) );
4464 for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
4465 if( j>=pLoop->nLTerm ) continue;
4466 }
4467 if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
4468 Parse *pParse = pWInfo->pParse;
4469 CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr);
4470 CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr);
4471 assert( pColl1 );
4472 if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){
4473 continue;
4474 }
4475 testcase( pTerm->pExpr->op==TK_IS );
4476 }
4477 obSat |= MASKBIT(i);
4478 }
4479
4480 if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
4481 if( pLoop->wsFlags & WHERE_IPK ){
4482 pIndex = 0;
4483 nKeyCol = 0;
4484 nColumn = 1;
4485 }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
4486 return 0;
4487 }else{
4488 nKeyCol = pIndex->nKeyCol;
4489 nColumn = pIndex->nColumn;
4490 assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) );
4491 assert( pIndex->aiColumn[nColumn-1]==XN_ROWID
4492 || !HasRowid(pIndex->pTable));
4493 /* All relevant terms of the index must also be non-NULL in order
4494 ** for isOrderDistinct to be true. So the isOrderDistint value
4495 ** computed here might be a false positive. Corrections will be
4496 ** made at tag-20210426-1 below */
4497 isOrderDistinct = IsUniqueIndex(pIndex)
4498 && (pLoop->wsFlags & WHERE_SKIPSCAN)==0;
4499 }
4500
4501 /* Loop through all columns of the index and deal with the ones
4502 ** that are not constrained by == or IN.
4503 */
4504 rev = revSet = 0;
4505 distinctColumns = 0;
4506 for(j=0; j<nColumn; j++){
4507 u8 bOnce = 1; /* True to run the ORDER BY search loop */
4508
4509 assert( j>=pLoop->u.btree.nEq
4510 || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip)
4511 );
4512 if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){
4513 u16 eOp = pLoop->aLTerm[j]->eOperator;
4514
4515 /* Skip over == and IS and ISNULL terms. (Also skip IN terms when
4516 ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL
4517 ** terms imply that the index is not UNIQUE NOT NULL in which case
4518 ** the loop need to be marked as not order-distinct because it can
4519 ** have repeated NULL rows.
4520 **
4521 ** If the current term is a column of an ((?,?) IN (SELECT...))
4522 ** expression for which the SELECT returns more than one column,
4523 ** check that it is the only column used by this loop. Otherwise,
4524 ** if it is one of two or more, none of the columns can be
4525 ** considered to match an ORDER BY term.
4526 */
4527 if( (eOp & eqOpMask)!=0 ){
4528 if( eOp & (WO_ISNULL|WO_IS) ){
4529 testcase( eOp & WO_ISNULL );
4530 testcase( eOp & WO_IS );
4531 testcase( isOrderDistinct );
4532 isOrderDistinct = 0;
4533 }
4534 continue;
4535 }else if( ALWAYS(eOp & WO_IN) ){
4536 /* ALWAYS() justification: eOp is an equality operator due to the
4537 ** j<pLoop->u.btree.nEq constraint above. Any equality other
4538 ** than WO_IN is captured by the previous "if". So this one
4539 ** always has to be WO_IN. */
4540 Expr *pX = pLoop->aLTerm[j]->pExpr;
4541 for(i=j+1; i<pLoop->u.btree.nEq; i++){
4542 if( pLoop->aLTerm[i]->pExpr==pX ){
4543 assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
4544 bOnce = 0;
4545 break;
4546 }
4547 }
4548 }
4549 }
4550
4551 /* Get the column number in the table (iColumn) and sort order
4552 ** (revIdx) for the j-th column of the index.
4553 */
4554 if( pIndex ){
4555 iColumn = pIndex->aiColumn[j];
4556 revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC;
4557 if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID;
4558 }else{
4559 iColumn = XN_ROWID;
4560 revIdx = 0;
4561 }
4562
4563 /* An unconstrained column that might be NULL means that this
4564 ** WhereLoop is not well-ordered. tag-20210426-1
4565 */
4566 if( isOrderDistinct ){
4567 if( iColumn>=0
4568 && j>=pLoop->u.btree.nEq
4569 && pIndex->pTable->aCol[iColumn].notNull==0
4570 ){
4571 isOrderDistinct = 0;
4572 }
4573 if( iColumn==XN_EXPR ){
4574 isOrderDistinct = 0;
4575 }
4576 }
4577
4578 /* Find the ORDER BY term that corresponds to the j-th column
4579 ** of the index and mark that ORDER BY term off
4580 */
4581 isMatch = 0;
4582 for(i=0; bOnce && i<nOrderBy; i++){
4583 if( MASKBIT(i) & obSat ) continue;
4584 pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
4585 testcase( wctrlFlags & WHERE_GROUPBY );
4586 testcase( wctrlFlags & WHERE_DISTINCTBY );
4587 if( NEVER(pOBExpr==0) ) continue;
4588 if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
4589 if( iColumn>=XN_ROWID ){
4590 if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
4591 if( pOBExpr->iTable!=iCur ) continue;
4592 if( pOBExpr->iColumn!=iColumn ) continue;
4593 }else{
4594 Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr;
4595 if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){
4596 continue;
4597 }
4598 }
4599 if( iColumn!=XN_ROWID ){
4600 pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
4601 if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue;
4602 }
4603 if( wctrlFlags & WHERE_DISTINCTBY ){
4604 pLoop->u.btree.nDistinctCol = j+1;
4605 }
4606 isMatch = 1;
4607 break;
4608 }
4609 if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
4610 /* Make sure the sort order is compatible in an ORDER BY clause.
4611 ** Sort order is irrelevant for a GROUP BY clause. */
4612 if( revSet ){
4613 if( (rev ^ revIdx)
4614 != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC)
4615 ){
4616 isMatch = 0;
4617 }
4618 }else{
4619 rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC);
4620 if( rev ) *pRevMask |= MASKBIT(iLoop);
4621 revSet = 1;
4622 }
4623 }
4624 if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){
4625 if( j==pLoop->u.btree.nEq ){
4626 pLoop->wsFlags |= WHERE_BIGNULL_SORT;
4627 }else{
4628 isMatch = 0;
4629 }
4630 }
4631 if( isMatch ){
4632 if( iColumn==XN_ROWID ){
4633 testcase( distinctColumns==0 );
4634 distinctColumns = 1;
4635 }
4636 obSat |= MASKBIT(i);
4637 }else{
4638 /* No match found */
4639 if( j==0 || j<nKeyCol ){
4640 testcase( isOrderDistinct!=0 );
4641 isOrderDistinct = 0;
4642 }
4643 break;
4644 }
4645 } /* end Loop over all index columns */
4646 if( distinctColumns ){
4647 testcase( isOrderDistinct==0 );
4648 isOrderDistinct = 1;
4649 }
4650 } /* end-if not one-row */
4651
4652 /* Mark off any other ORDER BY terms that reference pLoop */
4653 if( isOrderDistinct ){
4654 orderDistinctMask |= pLoop->maskSelf;
4655 for(i=0; i<nOrderBy; i++){
4656 Expr *p;
4657 Bitmask mTerm;
4658 if( MASKBIT(i) & obSat ) continue;
4659 p = pOrderBy->a[i].pExpr;
4660 mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p);
4661 if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue;
4662 if( (mTerm&~orderDistinctMask)==0 ){
4663 obSat |= MASKBIT(i);
4664 }
4665 }
4666 }
4667 } /* End the loop over all WhereLoops from outer-most down to inner-most */
4668 if( obSat==obDone ) return (i8)nOrderBy;
4669 if( !isOrderDistinct ){
4670 for(i=nOrderBy-1; i>0; i--){
4671 Bitmask m = ALWAYS(i<BMS) ? MASKBIT(i) - 1 : 0;
4672 if( (obSat&m)==m ) return i;
4673 }
4674 return 0;
4675 }
4676 return -1;
4677}
4678
4679
4680/*
4681** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(),
4682** the planner assumes that the specified pOrderBy list is actually a GROUP
4683** BY clause - and so any order that groups rows as required satisfies the
4684** request.
4685**
4686** Normally, in this case it is not possible for the caller to determine
4687** whether or not the rows are really being delivered in sorted order, or
4688** just in some other order that provides the required grouping. However,
4689** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then
4690** this function may be called on the returned WhereInfo object. It returns
4691** true if the rows really will be sorted in the specified order, or false
4692** otherwise.
4693**
4694** For example, assuming:
4695**
4696** CREATE INDEX i1 ON t1(x, Y);
4697**
4698** then
4699**
4700** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1
4701** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0
4702*/
4703int sqlite3WhereIsSorted(WhereInfo *pWInfo){
4704 assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) );
4705 assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP );
4706 return pWInfo->sorted;
4707}
4708
4709#ifdef WHERETRACE_ENABLED
4710/* For debugging use only: */
4711static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
4712 static char zName[65];
4713 int i;
4714 for(i=0; i<nLoop; i++){ zName[i] = pPath->aLoop[i]->cId; }
4715 if( pLast ) zName[i++] = pLast->cId;
4716 zName[i] = 0;
4717 return zName;
4718}
4719#endif
4720
4721/*
4722** Return the cost of sorting nRow rows, assuming that the keys have
4723** nOrderby columns and that the first nSorted columns are already in
4724** order.
4725*/
4726static LogEst whereSortingCost(
4727 WhereInfo *pWInfo,
4728 LogEst nRow,
4729 int nOrderBy,
4730 int nSorted
4731){
4732 /* TUNING: Estimated cost of a full external sort, where N is
4733 ** the number of rows to sort is:
4734 **
4735 ** cost = (3.0 * N * log(N)).
4736 **
4737 ** Or, if the order-by clause has X terms but only the last Y
4738 ** terms are out of order, then block-sorting will reduce the
4739 ** sorting cost to:
4740 **
4741 ** cost = (3.0 * N * log(N)) * (Y/X)
4742 **
4743 ** The (Y/X) term is implemented using stack variable rScale
4744 ** below.
4745 */
4746 LogEst rScale, rSortCost;
4747 assert( nOrderBy>0 && 66==sqlite3LogEst(100) );
4748 rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
4749 rSortCost = nRow + rScale + 16;
4750
4751 /* Multiple by log(M) where M is the number of output rows.
4752 ** Use the LIMIT for M if it is smaller. Or if this sort is for
4753 ** a DISTINCT operator, M will be the number of distinct output
4754 ** rows, so fudge it downwards a bit.
4755 */
4756 if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimit<nRow ){
4757 nRow = pWInfo->iLimit;
4758 }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){
4759 /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT
4760 ** reduces the number of output rows by a factor of 2 */
4761 if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); }
4762 }
4763 rSortCost += estLog(nRow);
4764 return rSortCost;
4765}
4766
4767/*
4768** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
4769** attempts to find the lowest cost path that visits each WhereLoop
4770** once. This path is then loaded into the pWInfo->a[].pWLoop fields.
4771**
4772** Assume that the total number of output rows that will need to be sorted
4773** will be nRowEst (in the 10*log2 representation). Or, ignore sorting
4774** costs if nRowEst==0.
4775**
4776** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation
4777** error occurs.
4778*/
4779static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
4780 int mxChoice; /* Maximum number of simultaneous paths tracked */
4781 int nLoop; /* Number of terms in the join */
4782 Parse *pParse; /* Parsing context */
4783 int iLoop; /* Loop counter over the terms of the join */
4784 int ii, jj; /* Loop counters */
4785 int mxI = 0; /* Index of next entry to replace */
4786 int nOrderBy; /* Number of ORDER BY clause terms */
4787 LogEst mxCost = 0; /* Maximum cost of a set of paths */
4788 LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */
4789 int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */
4790 WherePath *aFrom; /* All nFrom paths at the previous level */
4791 WherePath *aTo; /* The nTo best paths at the current level */
4792 WherePath *pFrom; /* An element of aFrom[] that we are working on */
4793 WherePath *pTo; /* An element of aTo[] that we are working on */
4794 WhereLoop *pWLoop; /* One of the WhereLoop objects */
4795 WhereLoop **pX; /* Used to divy up the pSpace memory */
4796 LogEst *aSortCost = 0; /* Sorting and partial sorting costs */
4797 char *pSpace; /* Temporary memory used by this routine */
4798 int nSpace; /* Bytes of space allocated at pSpace */
4799
4800 pParse = pWInfo->pParse;
4801 nLoop = pWInfo->nLevel;
4802 /* TUNING: For simple queries, only the best path is tracked.
4803 ** For 2-way joins, the 5 best paths are followed.
4804 ** For joins of 3 or more tables, track the 10 best paths */
4805 mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10);
4806 assert( nLoop<=pWInfo->pTabList->nSrc );
4807 WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst));
4808
4809 /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
4810 ** case the purpose of this call is to estimate the number of rows returned
4811 ** by the overall query. Once this estimate has been obtained, the caller
4812 ** will invoke this function a second time, passing the estimate as the
4813 ** nRowEst parameter. */
4814 if( pWInfo->pOrderBy==0 || nRowEst==0 ){
4815 nOrderBy = 0;
4816 }else{
4817 nOrderBy = pWInfo->pOrderBy->nExpr;
4818 }
4819
4820 /* Allocate and initialize space for aTo, aFrom and aSortCost[] */
4821 nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
4822 nSpace += sizeof(LogEst) * nOrderBy;
4823 pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace);
4824 if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
4825 aTo = (WherePath*)pSpace;
4826 aFrom = aTo+mxChoice;
4827 memset(aFrom, 0, sizeof(aFrom[0]));
4828 pX = (WhereLoop**)(aFrom+mxChoice);
4829 for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){
4830 pFrom->aLoop = pX;
4831 }
4832 if( nOrderBy ){
4833 /* If there is an ORDER BY clause and it is not being ignored, set up
4834 ** space for the aSortCost[] array. Each element of the aSortCost array
4835 ** is either zero - meaning it has not yet been initialized - or the
4836 ** cost of sorting nRowEst rows of data where the first X terms of
4837 ** the ORDER BY clause are already in order, where X is the array
4838 ** index. */
4839 aSortCost = (LogEst*)pX;
4840 memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
4841 }
4842 assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] );
4843 assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX );
4844
4845 /* Seed the search with a single WherePath containing zero WhereLoops.
4846 **
4847 ** TUNING: Do not let the number of iterations go above 28. If the cost
4848 ** of computing an automatic index is not paid back within the first 28
4849 ** rows, then do not use the automatic index. */
4850 aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) );
4851 nFrom = 1;
4852 assert( aFrom[0].isOrdered==0 );
4853 if( nOrderBy ){
4854 /* If nLoop is zero, then there are no FROM terms in the query. Since
4855 ** in this case the query may return a maximum of one row, the results
4856 ** are already in the requested order. Set isOrdered to nOrderBy to
4857 ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
4858 ** -1, indicating that the result set may or may not be ordered,
4859 ** depending on the loops added to the current plan. */
4860 aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
4861 }
4862
4863 /* Compute successively longer WherePaths using the previous generation
4864 ** of WherePaths as the basis for the next. Keep track of the mxChoice
4865 ** best paths at each generation */
4866 for(iLoop=0; iLoop<nLoop; iLoop++){
4867 nTo = 0;
4868 for(ii=0, pFrom=aFrom; ii<nFrom; ii++, pFrom++){
4869 for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
4870 LogEst nOut; /* Rows visited by (pFrom+pWLoop) */
4871 LogEst rCost; /* Cost of path (pFrom+pWLoop) */
4872 LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */
4873 i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */
4874 Bitmask maskNew; /* Mask of src visited by (..) */
4875 Bitmask revMask; /* Mask of rev-order loops for (..) */
4876
4877 if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
4878 if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
4879 if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){
4880 /* Do not use an automatic index if the this loop is expected
4881 ** to run less than 1.25 times. It is tempting to also exclude
4882 ** automatic index usage on an outer loop, but sometimes an automatic
4883 ** index is useful in the outer loop of a correlated subquery. */
4884 assert( 10==sqlite3LogEst(2) );
4885 continue;
4886 }
4887
4888 /* At this point, pWLoop is a candidate to be the next loop.
4889 ** Compute its cost */
4890 rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
4891 rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
4892 nOut = pFrom->nRow + pWLoop->nOut;
4893 maskNew = pFrom->maskLoop | pWLoop->maskSelf;
4894 isOrdered = pFrom->isOrdered;
4895 if( isOrdered<0 ){
4896 revMask = 0;
4897 isOrdered = wherePathSatisfiesOrderBy(pWInfo,
4898 pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
4899 iLoop, pWLoop, &revMask);
4900 }else{
4901 revMask = pFrom->revLoop;
4902 }
4903 if( isOrdered>=0 && isOrdered<nOrderBy ){
4904 if( aSortCost[isOrdered]==0 ){
4905 aSortCost[isOrdered] = whereSortingCost(
4906 pWInfo, nRowEst, nOrderBy, isOrdered
4907 );
4908 }
4909 /* TUNING: Add a small extra penalty (5) to sorting as an
4910 ** extra encouragment to the query planner to select a plan
4911 ** where the rows emerge in the correct order without any sorting
4912 ** required. */
4913 rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 5;
4914
4915 WHERETRACE(0x002,
4916 ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
4917 aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
4918 rUnsorted, rCost));
4919 }else{
4920 rCost = rUnsorted;
4921 rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */
4922 }
4923
4924 /* TUNING: A full-scan of a VIEW or subquery in the outer loop
4925 ** is not so bad. */
4926 if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){
4927 rCost += -10;
4928 nOut += -30;
4929 }
4930
4931 /* Check to see if pWLoop should be added to the set of
4932 ** mxChoice best-so-far paths.
4933 **
4934 ** First look for an existing path among best-so-far paths
4935 ** that covers the same set of loops and has the same isOrdered
4936 ** setting as the current path candidate.
4937 **
4938 ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
4939 ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
4940 ** of legal values for isOrdered, -1..64.
4941 */
4942 for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
4943 if( pTo->maskLoop==maskNew
4944 && ((pTo->isOrdered^isOrdered)&0x80)==0
4945 ){
4946 testcase( jj==nTo-1 );
4947 break;
4948 }
4949 }
4950 if( jj>=nTo ){
4951 /* None of the existing best-so-far paths match the candidate. */
4952 if( nTo>=mxChoice
4953 && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted))
4954 ){
4955 /* The current candidate is no better than any of the mxChoice
4956 ** paths currently in the best-so-far buffer. So discard
4957 ** this candidate as not viable. */
4958#ifdef WHERETRACE_ENABLED /* 0x4 */
4959 if( sqlite3WhereTrace&0x4 ){
4960 sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n",
4961 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
4962 isOrdered>=0 ? isOrdered+'0' : '?');
4963 }
4964#endif
4965 continue;
4966 }
4967 /* If we reach this points it means that the new candidate path
4968 ** needs to be added to the set of best-so-far paths. */
4969 if( nTo<mxChoice ){
4970 /* Increase the size of the aTo set by one */
4971 jj = nTo++;
4972 }else{
4973 /* New path replaces the prior worst to keep count below mxChoice */
4974 jj = mxI;
4975 }
4976 pTo = &aTo[jj];
4977#ifdef WHERETRACE_ENABLED /* 0x4 */
4978 if( sqlite3WhereTrace&0x4 ){
4979 sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n",
4980 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
4981 isOrdered>=0 ? isOrdered+'0' : '?');
4982 }
4983#endif
4984 }else{
4985 /* Control reaches here if best-so-far path pTo=aTo[jj] covers the
4986 ** same set of loops and has the same isOrdered setting as the
4987 ** candidate path. Check to see if the candidate should replace
4988 ** pTo or if the candidate should be skipped.
4989 **
4990 ** The conditional is an expanded vector comparison equivalent to:
4991 ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted)
4992 */
4993 if( pTo->rCost<rCost
4994 || (pTo->rCost==rCost
4995 && (pTo->nRow<nOut
4996 || (pTo->nRow==nOut && pTo->rUnsorted<=rUnsorted)
4997 )
4998 )
4999 ){
5000#ifdef WHERETRACE_ENABLED /* 0x4 */
5001 if( sqlite3WhereTrace&0x4 ){
5002 sqlite3DebugPrintf(
5003 "Skip %s cost=%-3d,%3d,%3d order=%c",
5004 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
5005 isOrdered>=0 ? isOrdered+'0' : '?');
5006 sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n",
5007 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
5008 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
5009 }
5010#endif
5011 /* Discard the candidate path from further consideration */
5012 testcase( pTo->rCost==rCost );
5013 continue;
5014 }
5015 testcase( pTo->rCost==rCost+1 );
5016 /* Control reaches here if the candidate path is better than the
5017 ** pTo path. Replace pTo with the candidate. */
5018#ifdef WHERETRACE_ENABLED /* 0x4 */
5019 if( sqlite3WhereTrace&0x4 ){
5020 sqlite3DebugPrintf(
5021 "Update %s cost=%-3d,%3d,%3d order=%c",
5022 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
5023 isOrdered>=0 ? isOrdered+'0' : '?');
5024 sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n",
5025 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
5026 pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
5027 }
5028#endif
5029 }
5030 /* pWLoop is a winner. Add it to the set of best so far */
5031 pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf;
5032 pTo->revLoop = revMask;
5033 pTo->nRow = nOut;
5034 pTo->rCost = rCost;
5035 pTo->rUnsorted = rUnsorted;
5036 pTo->isOrdered = isOrdered;
5037 memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
5038 pTo->aLoop[iLoop] = pWLoop;
5039 if( nTo>=mxChoice ){
5040 mxI = 0;
5041 mxCost = aTo[0].rCost;
5042 mxUnsorted = aTo[0].nRow;
5043 for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
5044 if( pTo->rCost>mxCost
5045 || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted)
5046 ){
5047 mxCost = pTo->rCost;
5048 mxUnsorted = pTo->rUnsorted;
5049 mxI = jj;
5050 }
5051 }
5052 }
5053 }
5054 }
5055
5056#ifdef WHERETRACE_ENABLED /* >=2 */
5057 if( sqlite3WhereTrace & 0x02 ){
5058 sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
5059 for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
5060 sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
5061 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
5062 pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
5063 if( pTo->isOrdered>0 ){
5064 sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
5065 }else{
5066 sqlite3DebugPrintf("\n");
5067 }
5068 }
5069 }
5070#endif
5071
5072 /* Swap the roles of aFrom and aTo for the next generation */
5073 pFrom = aTo;
5074 aTo = aFrom;
5075 aFrom = pFrom;
5076 nFrom = nTo;
5077 }
5078
5079 if( nFrom==0 ){
5080 sqlite3ErrorMsg(pParse, "no query solution");
5081 sqlite3StackFreeNN(pParse->db, pSpace);
5082 return SQLITE_ERROR;
5083 }
5084
5085 /* Find the lowest cost path. pFrom will be left pointing to that path */
5086 pFrom = aFrom;
5087 for(ii=1; ii<nFrom; ii++){
5088 if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii];
5089 }
5090 assert( pWInfo->nLevel==nLoop );
5091 /* Load the lowest cost path into pWInfo */
5092 for(iLoop=0; iLoop<nLoop; iLoop++){
5093 WhereLevel *pLevel = pWInfo->a + iLoop;
5094 pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop];
5095 pLevel->iFrom = pWLoop->iTab;
5096 pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor;
5097 }
5098 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0
5099 && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0
5100 && pWInfo->eDistinct==WHERE_DISTINCT_NOOP
5101 && nRowEst
5102 ){
5103 Bitmask notUsed;
5104 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom,
5105 WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], &notUsed);
5106 if( rc==pWInfo->pResultSet->nExpr ){
5107 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
5108 }
5109 }
5110 pWInfo->bOrderedInnerLoop = 0;
5111 if( pWInfo->pOrderBy ){
5112 pWInfo->nOBSat = pFrom->isOrdered;
5113 if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
5114 if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
5115 pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
5116 }
5117 }else{
5118 pWInfo->revMask = pFrom->revLoop;
5119 if( pWInfo->nOBSat<=0 ){
5120 pWInfo->nOBSat = 0;
5121 if( nLoop>0 ){
5122 u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
5123 if( (wsFlags & WHERE_ONEROW)==0
5124 && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
5125 ){
5126 Bitmask m = 0;
5127 int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
5128 WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
5129 testcase( wsFlags & WHERE_IPK );
5130 testcase( wsFlags & WHERE_COLUMN_IN );
5131 if( rc==pWInfo->pOrderBy->nExpr ){
5132 pWInfo->bOrderedInnerLoop = 1;
5133 pWInfo->revMask = m;
5134 }
5135 }
5136 }
5137 }else if( nLoop
5138 && pWInfo->nOBSat==1
5139 && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0
5140 ){
5141 pWInfo->bOrderedInnerLoop = 1;
5142 }
5143 }
5144 if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
5145 && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
5146 ){
5147 Bitmask revMask = 0;
5148 int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy,
5149 pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
5150 );
5151 assert( pWInfo->sorted==0 );
5152 if( nOrder==pWInfo->pOrderBy->nExpr ){
5153 pWInfo->sorted = 1;
5154 pWInfo->revMask = revMask;
5155 }
5156 }
5157 }
5158
5159
5160 pWInfo->nRowOut = pFrom->nRow;
5161
5162 /* Free temporary memory and return success */
5163 sqlite3StackFreeNN(pParse->db, pSpace);
5164 return SQLITE_OK;
5165}
5166
5167/*
5168** Most queries use only a single table (they are not joins) and have
5169** simple == constraints against indexed fields. This routine attempts
5170** to plan those simple cases using much less ceremony than the
5171** general-purpose query planner, and thereby yield faster sqlite3_prepare()
5172** times for the common case.
5173**
5174** Return non-zero on success, if this query can be handled by this
5175** no-frills query planner. Return zero if this query needs the
5176** general-purpose query planner.
5177*/
5178static int whereShortCut(WhereLoopBuilder *pBuilder){
5179 WhereInfo *pWInfo;
5180 SrcItem *pItem;
5181 WhereClause *pWC;
5182 WhereTerm *pTerm;
5183 WhereLoop *pLoop;
5184 int iCur;
5185 int j;
5186 Table *pTab;
5187 Index *pIdx;
5188 WhereScan scan;
5189
5190 pWInfo = pBuilder->pWInfo;
5191 if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
5192 assert( pWInfo->pTabList->nSrc>=1 );
5193 pItem = pWInfo->pTabList->a;
5194 pTab = pItem->pTab;
5195 if( IsVirtual(pTab) ) return 0;
5196 if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){
5197 testcase( pItem->fg.isIndexedBy );
5198 testcase( pItem->fg.notIndexed );
5199 return 0;
5200 }
5201 iCur = pItem->iCursor;
5202 pWC = &pWInfo->sWC;
5203 pLoop = pBuilder->pNew;
5204 pLoop->wsFlags = 0;
5205 pLoop->nSkip = 0;
5206 pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0);
5207 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
5208 if( pTerm ){
5209 testcase( pTerm->eOperator & WO_IS );
5210 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
5211 pLoop->aLTerm[0] = pTerm;
5212 pLoop->nLTerm = 1;
5213 pLoop->u.btree.nEq = 1;
5214 /* TUNING: Cost of a rowid lookup is 10 */
5215 pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */
5216 }else{
5217 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
5218 int opMask;
5219 assert( pLoop->aLTermSpace==pLoop->aLTerm );
5220 if( !IsUniqueIndex(pIdx)
5221 || pIdx->pPartIdxWhere!=0
5222 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace)
5223 ) continue;
5224 opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
5225 for(j=0; j<pIdx->nKeyCol; j++){
5226 pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx);
5227 while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan);
5228 if( pTerm==0 ) break;
5229 testcase( pTerm->eOperator & WO_IS );
5230 pLoop->aLTerm[j] = pTerm;
5231 }
5232 if( j!=pIdx->nKeyCol ) continue;
5233 pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED;
5234 if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){
5235 pLoop->wsFlags |= WHERE_IDX_ONLY;
5236 }
5237 pLoop->nLTerm = j;
5238 pLoop->u.btree.nEq = j;
5239 pLoop->u.btree.pIndex = pIdx;
5240 /* TUNING: Cost of a unique index lookup is 15 */
5241 pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */
5242 break;
5243 }
5244 }
5245 if( pLoop->wsFlags ){
5246 pLoop->nOut = (LogEst)1;
5247 pWInfo->a[0].pWLoop = pLoop;
5248 assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] );
5249 pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */
5250 pWInfo->a[0].iTabCur = iCur;
5251 pWInfo->nRowOut = 1;
5252 if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr;
5253 if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){
5254 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
5255 }
5256 if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS;
5257#ifdef SQLITE_DEBUG
5258 pLoop->cId = '0';
5259#endif
5260#ifdef WHERETRACE_ENABLED
5261 if( sqlite3WhereTrace ){
5262 sqlite3DebugPrintf("whereShortCut() used to compute solution\n");
5263 }
5264#endif
5265 return 1;
5266 }
5267 return 0;
5268}
5269
5270/*
5271** Helper function for exprIsDeterministic().
5272*/
5273static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){
5274 if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){
5275 pWalker->eCode = 0;
5276 return WRC_Abort;
5277 }
5278 return WRC_Continue;
5279}
5280
5281/*
5282** Return true if the expression contains no non-deterministic SQL
5283** functions. Do not consider non-deterministic SQL functions that are
5284** part of sub-select statements.
5285*/
5286static int exprIsDeterministic(Expr *p){
5287 Walker w;
5288 memset(&w, 0, sizeof(w));
5289 w.eCode = 1;
5290 w.xExprCallback = exprNodeIsDeterministic;
5291 w.xSelectCallback = sqlite3SelectWalkFail;
5292 sqlite3WalkExpr(&w, p);
5293 return w.eCode;
5294}
5295
5296
5297#ifdef WHERETRACE_ENABLED
5298/*
5299** Display all WhereLoops in pWInfo
5300*/
5301static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
5302 if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */
5303 WhereLoop *p;
5304 int i;
5305 static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
5306 "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
5307 for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
5308 p->cId = zLabel[i%(sizeof(zLabel)-1)];
5309 sqlite3WhereLoopPrint(p, pWC);
5310 }
5311 }
5312}
5313# define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C)
5314#else
5315# define WHERETRACE_ALL_LOOPS(W,C)
5316#endif
5317
5318/* Attempt to omit tables from a join that do not affect the result.
5319** For a table to not affect the result, the following must be true:
5320**
5321** 1) The query must not be an aggregate.
5322** 2) The table must be the RHS of a LEFT JOIN.
5323** 3) Either the query must be DISTINCT, or else the ON or USING clause
5324** must contain a constraint that limits the scan of the table to
5325** at most a single row.
5326** 4) The table must not be referenced by any part of the query apart
5327** from its own USING or ON clause.
5328**
5329** For example, given:
5330**
5331** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1);
5332** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2);
5333** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3);
5334**
5335** then table t2 can be omitted from the following:
5336**
5337** SELECT v1, v3 FROM t1
5338** LEFT JOIN t2 ON (t1.ipk=t2.ipk)
5339** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
5340**
5341** or from:
5342**
5343** SELECT DISTINCT v1, v3 FROM t1
5344** LEFT JOIN t2
5345** LEFT JOIN t3 ON (t1.ipk=t3.ipk)
5346*/
5347static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
5348 WhereInfo *pWInfo,
5349 Bitmask notReady
5350){
5351 int i;
5352 Bitmask tabUsed;
5353
5354 /* Preconditions checked by the caller */
5355 assert( pWInfo->nLevel>=2 );
5356 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) );
5357
5358 /* These two preconditions checked by the caller combine to guarantee
5359 ** condition (1) of the header comment */
5360 assert( pWInfo->pResultSet!=0 );
5361 assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) );
5362
5363 tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet);
5364 if( pWInfo->pOrderBy ){
5365 tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy);
5366 }
5367 for(i=pWInfo->nLevel-1; i>=1; i--){
5368 WhereTerm *pTerm, *pEnd;
5369 SrcItem *pItem;
5370 WhereLoop *pLoop;
5371 pLoop = pWInfo->a[i].pWLoop;
5372 pItem = &pWInfo->pTabList->a[pLoop->iTab];
5373 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue;
5374 if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0
5375 && (pLoop->wsFlags & WHERE_ONEROW)==0
5376 ){
5377 continue;
5378 }
5379 if( (tabUsed & pLoop->maskSelf)!=0 ) continue;
5380 pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm;
5381 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
5382 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
5383 if( !ExprHasProperty(pTerm->pExpr, EP_OuterON)
5384 || pTerm->pExpr->w.iJoin!=pItem->iCursor
5385 ){
5386 break;
5387 }
5388 }
5389 }
5390 if( pTerm<pEnd ) continue;
5391 WHERETRACE(0xffff, ("-> drop loop %c not used\n", pLoop->cId));
5392 notReady &= ~pLoop->maskSelf;
5393 for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
5394 if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
5395 pTerm->wtFlags |= TERM_CODED;
5396 }
5397 }
5398 if( i!=pWInfo->nLevel-1 ){
5399 int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel);
5400 memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte);
5401 }
5402 pWInfo->nLevel--;
5403 assert( pWInfo->nLevel>0 );
5404 }
5405 return notReady;
5406}
5407
5408/*
5409** Check to see if there are any SEARCH loops that might benefit from
5410** using a Bloom filter. Consider a Bloom filter if:
5411**
5412** (1) The SEARCH happens more than N times where N is the number
5413** of rows in the table that is being considered for the Bloom
5414** filter.
5415** (2) Some searches are expected to find zero rows. (This is determined
5416** by the WHERE_SELFCULL flag on the term.)
5417** (3) Bloom-filter processing is not disabled. (Checked by the
5418** caller.)
5419** (4) The size of the table being searched is known by ANALYZE.
5420**
5421** This block of code merely checks to see if a Bloom filter would be
5422** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the
5423** WhereLoop. The implementation of the Bloom filter comes further
5424** down where the code for each WhereLoop is generated.
5425*/
5426static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
5427 const WhereInfo *pWInfo
5428){
5429 int i;
5430 LogEst nSearch;
5431
5432 assert( pWInfo->nLevel>=2 );
5433 assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) );
5434 nSearch = pWInfo->a[0].pWLoop->nOut;
5435 for(i=1; i<pWInfo->nLevel; i++){
5436 WhereLoop *pLoop = pWInfo->a[i].pWLoop;
5437 const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
5438 if( (pLoop->wsFlags & reqFlags)==reqFlags
5439 /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
5440 && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0)
5441 ){
5442 SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
5443 Table *pTab = pItem->pTab;
5444 pTab->tabFlags |= TF_StatsUsed;
5445 if( nSearch > pTab->nRowLogEst
5446 && (pTab->tabFlags & TF_HasStat1)!=0
5447 ){
5448 testcase( pItem->fg.jointype & JT_LEFT );
5449 pLoop->wsFlags |= WHERE_BLOOMFILTER;
5450 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
5451 WHERETRACE(0xffff, (
5452 "-> use Bloom-filter on loop %c because there are ~%.1e "
5453 "lookups into %s which has only ~%.1e rows\n",
5454 pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName,
5455 (double)sqlite3LogEstToInt(pTab->nRowLogEst)));
5456 }
5457 }
5458 nSearch += pLoop->nOut;
5459 }
5460}
5461
5462/*
5463** This is an sqlite3ParserAddCleanup() callback that is invoked to
5464** free the Parse->pIdxExpr list when the Parse object is destroyed.
5465*/
5466static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
5467 Parse *pParse = (Parse*)pObject;
5468 while( pParse->pIdxExpr!=0 ){
5469 IndexedExpr *p = pParse->pIdxExpr;
5470 pParse->pIdxExpr = p->pIENext;
5471 sqlite3ExprDelete(db, p->pExpr);
5472 sqlite3DbFreeNN(db, p);
5473 }
5474}
5475
5476/*
5477** The index pIdx is used by a query and contains one or more expressions.
5478** In other words pIdx is an index on an expression. iIdxCur is the cursor
5479** number for the index and iDataCur is the cursor number for the corresponding
5480** table.
5481**
5482** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for
5483** each of the expressions in the index so that the expression code generator
5484** will know to replace occurrences of the indexed expression with
5485** references to the corresponding column of the index.
5486*/
5487static SQLITE_NOINLINE void whereAddIndexedExpr(
5488 Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */
5489 Index *pIdx, /* The index-on-expression that contains the expressions */
5490 int iIdxCur, /* Cursor number for pIdx */
5491 SrcItem *pTabItem /* The FROM clause entry for the table */
5492){
5493 int i;
5494 IndexedExpr *p;
5495 Table *pTab;
5496 assert( pIdx->bHasExpr );
5497 pTab = pIdx->pTable;
5498 for(i=0; i<pIdx->nColumn; i++){
5499 Expr *pExpr;
5500 int j = pIdx->aiColumn[i];
5501 int bMaybeNullRow;
5502 if( j==XN_EXPR ){
5503 pExpr = pIdx->aColExpr->a[i].pExpr;
5504 testcase( pTabItem->fg.jointype & JT_LEFT );
5505 testcase( pTabItem->fg.jointype & JT_RIGHT );
5506 testcase( pTabItem->fg.jointype & JT_LTORJ );
5507 bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
5508 }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){
5509 pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]);
5510 bMaybeNullRow = 0;
5511 }else{
5512 continue;
5513 }
5514 if( sqlite3ExprIsConstant(pExpr) ) continue;
5515 p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr));
5516 if( p==0 ) break;
5517 p->pIENext = pParse->pIdxExpr;
5518 p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0);
5519 p->iDataCur = pTabItem->iCursor;
5520 p->iIdxCur = iIdxCur;
5521 p->iIdxCol = i;
5522 p->bMaybeNullRow = bMaybeNullRow;
5523#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
5524 p->zIdxName = pIdx->zName;
5525#endif
5526 pParse->pIdxExpr = p;
5527 if( p->pIENext==0 ){
5528 sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse);
5529 }
5530 }
5531}
5532
5533/*
5534** Generate the beginning of the loop used for WHERE clause processing.
5535** The return value is a pointer to an opaque structure that contains
5536** information needed to terminate the loop. Later, the calling routine
5537** should invoke sqlite3WhereEnd() with the return value of this function
5538** in order to complete the WHERE clause processing.
5539**
5540** If an error occurs, this routine returns NULL.
5541**
5542** The basic idea is to do a nested loop, one loop for each table in
5543** the FROM clause of a select. (INSERT and UPDATE statements are the
5544** same as a SELECT with only a single table in the FROM clause.) For
5545** example, if the SQL is this:
5546**
5547** SELECT * FROM t1, t2, t3 WHERE ...;
5548**
5549** Then the code generated is conceptually like the following:
5550**
5551** foreach row1 in t1 do \ Code generated
5552** foreach row2 in t2 do |-- by sqlite3WhereBegin()
5553** foreach row3 in t3 do /
5554** ...
5555** end \ Code generated
5556** end |-- by sqlite3WhereEnd()
5557** end /
5558**
5559** Note that the loops might not be nested in the order in which they
5560** appear in the FROM clause if a different order is better able to make
5561** use of indices. Note also that when the IN operator appears in
5562** the WHERE clause, it might result in additional nested loops for
5563** scanning through all values on the right-hand side of the IN.
5564**
5565** There are Btree cursors associated with each table. t1 uses cursor
5566** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor.
5567** And so forth. This routine generates code to open those VDBE cursors
5568** and sqlite3WhereEnd() generates the code to close them.
5569**
5570** The code that sqlite3WhereBegin() generates leaves the cursors named
5571** in pTabList pointing at their appropriate entries. The [...] code
5572** can use OP_Column and OP_Rowid opcodes on these cursors to extract
5573** data from the various tables of the loop.
5574**
5575** If the WHERE clause is empty, the foreach loops must each scan their
5576** entire tables. Thus a three-way join is an O(N^3) operation. But if
5577** the tables have indices and there are terms in the WHERE clause that
5578** refer to those indices, a complete table scan can be avoided and the
5579** code will run much faster. Most of the work of this routine is checking
5580** to see if there are indices that can be used to speed up the loop.
5581**
5582** Terms of the WHERE clause are also used to limit which rows actually
5583** make it to the "..." in the middle of the loop. After each "foreach",
5584** terms of the WHERE clause that use only terms in that loop and outer
5585** loops are evaluated and if false a jump is made around all subsequent
5586** inner loops (or around the "..." if the test occurs within the inner-
5587** most loop)
5588**
5589** OUTER JOINS
5590**
5591** An outer join of tables t1 and t2 is conceptally coded as follows:
5592**
5593** foreach row1 in t1 do
5594** flag = 0
5595** foreach row2 in t2 do
5596** start:
5597** ...
5598** flag = 1
5599** end
5600** if flag==0 then
5601** move the row2 cursor to a null row
5602** goto start
5603** fi
5604** end
5605**
5606** ORDER BY CLAUSE PROCESSING
5607**
5608** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause
5609** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement
5610** if there is one. If there is no ORDER BY clause or if this routine
5611** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
5612**
5613** The iIdxCur parameter is the cursor number of an index. If
5614** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
5615** to use for OR clause processing. The WHERE clause should use this
5616** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
5617** the first cursor in an array of cursors for all indices. iIdxCur should
5618** be used to compute the appropriate cursor depending on which index is
5619** used.
5620*/
5621WhereInfo *sqlite3WhereBegin(
5622 Parse *pParse, /* The parser context */
5623 SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */
5624 Expr *pWhere, /* The WHERE clause */
5625 ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */
5626 ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */
5627 Select *pSelect, /* The entire SELECT statement */
5628 u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */
5629 int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number
5630 ** If WHERE_USE_LIMIT, then the limit amount */
5631){
5632 int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
5633 int nTabList; /* Number of elements in pTabList */
5634 WhereInfo *pWInfo; /* Will become the return value of this function */
5635 Vdbe *v = pParse->pVdbe; /* The virtual database engine */
5636 Bitmask notReady; /* Cursors that are not yet positioned */
5637 WhereLoopBuilder sWLB; /* The WhereLoop builder */
5638 WhereMaskSet *pMaskSet; /* The expression mask set */
5639 WhereLevel *pLevel; /* A single level in pWInfo->a[] */
5640 WhereLoop *pLoop; /* Pointer to a single WhereLoop object */
5641 int ii; /* Loop counter */
5642 sqlite3 *db; /* Database connection */
5643 int rc; /* Return code */
5644 u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */
5645
5646 assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
5647 (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0
5648 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
5649 ));
5650
5651 /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
5652 assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
5653 || (wctrlFlags & WHERE_USE_LIMIT)==0 );
5654
5655 /* Variable initialization */
5656 db = pParse->db;
5657 memset(&sWLB, 0, sizeof(sWLB));
5658
5659 /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
5660 testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
5661 if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0;
5662
5663 /* The number of tables in the FROM clause is limited by the number of
5664 ** bits in a Bitmask
5665 */
5666 testcase( pTabList->nSrc==BMS );
5667 if( pTabList->nSrc>BMS ){
5668 sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS);
5669 return 0;
5670 }
5671
5672 /* This function normally generates a nested loop for all tables in
5673 ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should
5674 ** only generate code for the first table in pTabList and assume that
5675 ** any cursors associated with subsequent tables are uninitialized.
5676 */
5677 nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
5678
5679 /* Allocate and initialize the WhereInfo structure that will become the
5680 ** return value. A single allocation is used to store the WhereInfo
5681 ** struct, the contents of WhereInfo.a[], the WhereClause structure
5682 ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte
5683 ** field (type Bitmask) it must be aligned on an 8-byte boundary on
5684 ** some architectures. Hence the ROUND8() below.
5685 */
5686 nByteWInfo = ROUND8P(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel));
5687 pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
5688 if( db->mallocFailed ){
5689 sqlite3DbFree(db, pWInfo);
5690 pWInfo = 0;
5691 goto whereBeginError;
5692 }
5693 pWInfo->pParse = pParse;
5694 pWInfo->pTabList = pTabList;
5695 pWInfo->pOrderBy = pOrderBy;
5696#if WHERETRACE_ENABLED
5697 pWInfo->pWhere = pWhere;
5698#endif
5699 pWInfo->pResultSet = pResultSet;
5700 pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
5701 pWInfo->nLevel = nTabList;
5702 pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse);
5703 pWInfo->wctrlFlags = wctrlFlags;
5704 pWInfo->iLimit = iAuxArg;
5705 pWInfo->savedNQueryLoop = pParse->nQueryLoop;
5706 pWInfo->pSelect = pSelect;
5707 memset(&pWInfo->nOBSat, 0,
5708 offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
5709 memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
5710 assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */
5711 pMaskSet = &pWInfo->sMaskSet;
5712 pMaskSet->n = 0;
5713 pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be
5714 ** a valid cursor number, to avoid an initial
5715 ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */
5716 sWLB.pWInfo = pWInfo;
5717 sWLB.pWC = &pWInfo->sWC;
5718 sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo);
5719 assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) );
5720 whereLoopInit(sWLB.pNew);
5721#ifdef SQLITE_DEBUG
5722 sWLB.pNew->cId = '*';
5723#endif
5724
5725 /* Split the WHERE clause into separate subexpressions where each
5726 ** subexpression is separated by an AND operator.
5727 */
5728 sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo);
5729 sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND);
5730
5731 /* Special case: No FROM clause
5732 */
5733 if( nTabList==0 ){
5734 if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr;
5735 if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0
5736 && OptimizationEnabled(db, SQLITE_DistinctOpt)
5737 ){
5738 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
5739 }
5740 ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
5741 }else{
5742 /* Assign a bit from the bitmask to every term in the FROM clause.
5743 **
5744 ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
5745 **
5746 ** The rule of the previous sentence ensures thta if X is the bitmask for
5747 ** a table T, then X-1 is the bitmask for all other tables to the left of T.
5748 ** Knowing the bitmask for all tables to the left of a left join is
5749 ** important. Ticket #3015.
5750 **
5751 ** Note that bitmasks are created for all pTabList->nSrc tables in
5752 ** pTabList, not just the first nTabList tables. nTabList is normally
5753 ** equal to pTabList->nSrc but might be shortened to 1 if the
5754 ** WHERE_OR_SUBCLAUSE flag is set.
5755 */
5756 ii = 0;
5757 do{
5758 createMask(pMaskSet, pTabList->a[ii].iCursor);
5759 sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC);
5760 }while( (++ii)<pTabList->nSrc );
5761 #ifdef SQLITE_DEBUG
5762 {
5763 Bitmask mx = 0;
5764 for(ii=0; ii<pTabList->nSrc; ii++){
5765 Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor);
5766 assert( m>=mx );
5767 mx = m;
5768 }
5769 }
5770 #endif
5771 }
5772
5773 /* Analyze all of the subexpressions. */
5774 sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
5775 if( pSelect && pSelect->pLimit ){
5776 sqlite3WhereAddLimit(&pWInfo->sWC, pSelect);
5777 }
5778 if( pParse->nErr ) goto whereBeginError;
5779
5780 /* Special case: WHERE terms that do not refer to any tables in the join
5781 ** (constant expressions). Evaluate each such term, and jump over all the
5782 ** generated code if the result is not true.
5783 **
5784 ** Do not do this if the expression contains non-deterministic functions
5785 ** that are not within a sub-select. This is not strictly required, but
5786 ** preserves SQLite's legacy behaviour in the following two cases:
5787 **
5788 ** FROM ... WHERE random()>0; -- eval random() once per row
5789 ** FROM ... WHERE (SELECT random())>0; -- eval random() once overall
5790 */
5791 for(ii=0; ii<sWLB.pWC->nBase; ii++){
5792 WhereTerm *pT = &sWLB.pWC->a[ii];
5793 if( pT->wtFlags & TERM_VIRTUAL ) continue;
5794 if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){
5795 sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL);
5796 pT->wtFlags |= TERM_CODED;
5797 }
5798 }
5799
5800 if( wctrlFlags & WHERE_WANT_DISTINCT ){
5801 if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){
5802 /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
5803 ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
5804 wctrlFlags &= ~WHERE_WANT_DISTINCT;
5805 pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT;
5806 }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){
5807 /* The DISTINCT marking is pointless. Ignore it. */
5808 pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
5809 }else if( pOrderBy==0 ){
5810 /* Try to ORDER BY the result set to make distinct processing easier */
5811 pWInfo->wctrlFlags |= WHERE_DISTINCTBY;
5812 pWInfo->pOrderBy = pResultSet;
5813 }
5814 }
5815
5816 /* Construct the WhereLoop objects */
5817#if defined(WHERETRACE_ENABLED)
5818 if( sqlite3WhereTrace & 0xffff ){
5819 sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags);
5820 if( wctrlFlags & WHERE_USE_LIMIT ){
5821 sqlite3DebugPrintf(", limit: %d", iAuxArg);
5822 }
5823 sqlite3DebugPrintf(")\n");
5824 if( sqlite3WhereTrace & 0x100 ){
5825 Select sSelect;
5826 memset(&sSelect, 0, sizeof(sSelect));
5827 sSelect.selFlags = SF_WhereBegin;
5828 sSelect.pSrc = pTabList;
5829 sSelect.pWhere = pWhere;
5830 sSelect.pOrderBy = pOrderBy;
5831 sSelect.pEList = pResultSet;
5832 sqlite3TreeViewSelect(0, &sSelect, 0);
5833 }
5834 }
5835 if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
5836 sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n");
5837 sqlite3WhereClausePrint(sWLB.pWC);
5838 }
5839#endif
5840
5841 if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
5842 rc = whereLoopAddAll(&sWLB);
5843 if( rc ) goto whereBeginError;
5844
5845#ifdef SQLITE_ENABLE_STAT4
5846 /* If one or more WhereTerm.truthProb values were used in estimating
5847 ** loop parameters, but then those truthProb values were subsequently
5848 ** changed based on STAT4 information while computing subsequent loops,
5849 ** then we need to rerun the whole loop building process so that all
5850 ** loops will be built using the revised truthProb values. */
5851 if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
5852 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
5853 WHERETRACE(0xffff,
5854 ("**** Redo all loop computations due to"
5855 " TERM_HIGHTRUTH changes ****\n"));
5856 while( pWInfo->pLoops ){
5857 WhereLoop *p = pWInfo->pLoops;
5858 pWInfo->pLoops = p->pNextLoop;
5859 whereLoopDelete(db, p);
5860 }
5861 rc = whereLoopAddAll(&sWLB);
5862 if( rc ) goto whereBeginError;
5863 }
5864#endif
5865 WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
5866
5867 wherePathSolver(pWInfo, 0);
5868 if( db->mallocFailed ) goto whereBeginError;
5869 if( pWInfo->pOrderBy ){
5870 wherePathSolver(pWInfo, pWInfo->nRowOut+1);
5871 if( db->mallocFailed ) goto whereBeginError;
5872 }
5873 }
5874 if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
5875 pWInfo->revMask = ALLBITS;
5876 }
5877 if( pParse->nErr ){
5878 goto whereBeginError;
5879 }
5880 assert( db->mallocFailed==0 );
5881#ifdef WHERETRACE_ENABLED
5882 if( sqlite3WhereTrace ){
5883 sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut);
5884 if( pWInfo->nOBSat>0 ){
5885 sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask);
5886 }
5887 switch( pWInfo->eDistinct ){
5888 case WHERE_DISTINCT_UNIQUE: {
5889 sqlite3DebugPrintf(" DISTINCT=unique");
5890 break;
5891 }
5892 case WHERE_DISTINCT_ORDERED: {
5893 sqlite3DebugPrintf(" DISTINCT=ordered");
5894 break;
5895 }
5896 case WHERE_DISTINCT_UNORDERED: {
5897 sqlite3DebugPrintf(" DISTINCT=unordered");
5898 break;
5899 }
5900 }
5901 sqlite3DebugPrintf("\n");
5902 for(ii=0; ii<pWInfo->nLevel; ii++){
5903 sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC);
5904 }
5905 }
5906#endif
5907
5908 /* Attempt to omit tables from a join that do not affect the result.
5909 ** See the comment on whereOmitNoopJoin() for further information.
5910 **
5911 ** This query optimization is factored out into a separate "no-inline"
5912 ** procedure to keep the sqlite3WhereBegin() procedure from becoming
5913 ** too large. If sqlite3WhereBegin() becomes too large, that prevents
5914 ** some C-compiler optimizers from in-lining the
5915 ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to
5916 ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons.
5917 */
5918 notReady = ~(Bitmask)0;
5919 if( pWInfo->nLevel>=2
5920 && pResultSet!=0 /* these two combine to guarantee */
5921 && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */
5922 && OptimizationEnabled(db, SQLITE_OmitNoopJoin)
5923 ){
5924 notReady = whereOmitNoopJoin(pWInfo, notReady);
5925 nTabList = pWInfo->nLevel;
5926 assert( nTabList>0 );
5927 }
5928
5929 /* Check to see if there are any SEARCH loops that might benefit from
5930 ** using a Bloom filter.
5931 */
5932 if( pWInfo->nLevel>=2
5933 && OptimizationEnabled(db, SQLITE_BloomFilter)
5934 ){
5935 whereCheckIfBloomFilterIsUseful(pWInfo);
5936 }
5937
5938#if defined(WHERETRACE_ENABLED)
5939 if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
5940 sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n");
5941 sqlite3WhereClausePrint(sWLB.pWC);
5942 }
5943 WHERETRACE(0xffff,("*** Optimizer Finished ***\n"));
5944#endif
5945 pWInfo->pParse->nQueryLoop += pWInfo->nRowOut;
5946
5947 /* If the caller is an UPDATE or DELETE statement that is requesting
5948 ** to use a one-pass algorithm, determine if this is appropriate.
5949 **
5950 ** A one-pass approach can be used if the caller has requested one
5951 ** and either (a) the scan visits at most one row or (b) each
5952 ** of the following are true:
5953 **
5954 ** * the caller has indicated that a one-pass approach can be used
5955 ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and
5956 ** * the table is not a virtual table, and
5957 ** * either the scan does not use the OR optimization or the caller
5958 ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified
5959 ** for DELETE).
5960 **
5961 ** The last qualification is because an UPDATE statement uses
5962 ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can
5963 ** use a one-pass approach, and this is not set accurately for scans
5964 ** that use the OR optimization.
5965 */
5966 assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 );
5967 if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
5968 int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
5969 int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
5970 assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) );
5971 if( bOnerow || (
5972 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
5973 && !IsVirtual(pTabList->a[0].pTab)
5974 && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
5975 )){
5976 pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
5977 if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){
5978 if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
5979 bFordelete = OPFLAG_FORDELETE;
5980 }
5981 pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY);
5982 }
5983 }
5984 }
5985
5986 /* Open all tables in the pTabList and any indices selected for
5987 ** searching those tables.
5988 */
5989 for(ii=0, pLevel=pWInfo->a; ii<nTabList; ii++, pLevel++){
5990 Table *pTab; /* Table to open */
5991 int iDb; /* Index of database containing table/index */
5992 SrcItem *pTabItem;
5993
5994 pTabItem = &pTabList->a[pLevel->iFrom];
5995 pTab = pTabItem->pTab;
5996 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
5997 pLoop = pLevel->pWLoop;
5998 if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){
5999 /* Do nothing */
6000 }else
6001#ifndef SQLITE_OMIT_VIRTUALTABLE
6002 if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
6003 const char *pVTab = (const char *)sqlite3GetVTable(db, pTab);
6004 int iCur = pTabItem->iCursor;
6005 sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB);
6006 }else if( IsVirtual(pTab) ){
6007 /* noop */
6008 }else
6009#endif
6010 if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0
6011 && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0)
6012 || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0
6013 ){
6014 int op = OP_OpenRead;
6015 if( pWInfo->eOnePass!=ONEPASS_OFF ){
6016 op = OP_OpenWrite;
6017 pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
6018 };
6019 sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
6020 assert( pTabItem->iCursor==pLevel->iTabCur );
6021 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
6022 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
6023 if( pWInfo->eOnePass==ONEPASS_OFF
6024 && pTab->nCol<BMS
6025 && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0
6026 && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0
6027 ){
6028 /* If we know that only a prefix of the record will be used,
6029 ** it is advantageous to reduce the "column count" field in
6030 ** the P4 operand of the OP_OpenRead/Write opcode. */
6031 Bitmask b = pTabItem->colUsed;
6032 int n = 0;
6033 for(; b; b=b>>1, n++){}
6034 sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32);
6035 assert( n<=pTab->nCol );
6036 }
6037#ifdef SQLITE_ENABLE_CURSOR_HINTS
6038 if( pLoop->u.btree.pIndex!=0 ){
6039 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
6040 }else
6041#endif
6042 {
6043 sqlite3VdbeChangeP5(v, bFordelete);
6044 }
6045#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
6046 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
6047 (const u8*)&pTabItem->colUsed, P4_INT64);
6048#endif
6049 }else{
6050 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
6051 }
6052 if( pLoop->wsFlags & WHERE_INDEXED ){
6053 Index *pIx = pLoop->u.btree.pIndex;
6054 int iIndexCur;
6055 int op = OP_OpenRead;
6056 /* iAuxArg is always set to a positive value if ONEPASS is possible */
6057 assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
6058 if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
6059 && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
6060 ){
6061 /* This is one term of an OR-optimization using the PRIMARY KEY of a
6062 ** WITHOUT ROWID table. No need for a separate index */
6063 iIndexCur = pLevel->iTabCur;
6064 op = 0;
6065 }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
6066 Index *pJ = pTabItem->pTab->pIndex;
6067 iIndexCur = iAuxArg;
6068 assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
6069 while( ALWAYS(pJ) && pJ!=pIx ){
6070 iIndexCur++;
6071 pJ = pJ->pNext;
6072 }
6073 op = OP_OpenWrite;
6074 pWInfo->aiCurOnePass[1] = iIndexCur;
6075 }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
6076 iIndexCur = iAuxArg;
6077 op = OP_ReopenIdx;
6078 }else{
6079 iIndexCur = pParse->nTab++;
6080 if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){
6081 whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem);
6082 }
6083 }
6084 pLevel->iIdxCur = iIndexCur;
6085 assert( pIx!=0 );
6086 assert( pIx->pSchema==pTab->pSchema );
6087 assert( iIndexCur>=0 );
6088 if( op ){
6089 sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb);
6090 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
6091 if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0
6092 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0
6093 && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0
6094 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
6095 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0
6096 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED
6097 ){
6098 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ);
6099 }
6100 VdbeComment((v, "%s", pIx->zName));
6101#ifdef SQLITE_ENABLE_COLUMN_USED_MASK
6102 {
6103 u64 colUsed = 0;
6104 int ii, jj;
6105 for(ii=0; ii<pIx->nColumn; ii++){
6106 jj = pIx->aiColumn[ii];
6107 if( jj<0 ) continue;
6108 if( jj>63 ) jj = 63;
6109 if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue;
6110 colUsed |= ((u64)1)<<(ii<63 ? ii : 63);
6111 }
6112 sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0,
6113 (u8*)&colUsed, P4_INT64);
6114 }
6115#endif /* SQLITE_ENABLE_COLUMN_USED_MASK */
6116 }
6117 }
6118 if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb);
6119 if( (pTabItem->fg.jointype & JT_RIGHT)!=0
6120 && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0
6121 ){
6122 WhereRightJoin *pRJ = pLevel->pRJ;
6123 pRJ->iMatch = pParse->nTab++;
6124 pRJ->regBloom = ++pParse->nMem;
6125 sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom);
6126 pRJ->regReturn = ++pParse->nMem;
6127 sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn);
6128 assert( pTab==pTabItem->pTab );
6129 if( HasRowid(pTab) ){
6130 KeyInfo *pInfo;
6131 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1);
6132 pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0);
6133 if( pInfo ){
6134 pInfo->aColl[0] = 0;
6135 pInfo->aSortFlags[0] = 0;
6136 sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO);
6137 }
6138 }else{
6139 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
6140 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol);
6141 sqlite3VdbeSetP4KeyInfo(pParse, pPk);
6142 }
6143 pLoop->wsFlags &= ~WHERE_IDX_ONLY;
6144 /* The nature of RIGHT JOIN processing is such that it messes up
6145 ** the output order. So omit any ORDER BY/GROUP BY elimination
6146 ** optimizations. We need to do an actual sort for RIGHT JOIN. */
6147 pWInfo->nOBSat = 0;
6148 pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED;
6149 }
6150 }
6151 pWInfo->iTop = sqlite3VdbeCurrentAddr(v);
6152 if( db->mallocFailed ) goto whereBeginError;
6153
6154 /* Generate the code to do the search. Each iteration of the for
6155 ** loop below generates code for a single nested loop of the VM
6156 ** program.
6157 */
6158 for(ii=0; ii<nTabList; ii++){
6159 int addrExplain;
6160 int wsFlags;
6161 SrcItem *pSrc;
6162 if( pParse->nErr ) goto whereBeginError;
6163 pLevel = &pWInfo->a[ii];
6164 wsFlags = pLevel->pWLoop->wsFlags;
6165 pSrc = &pTabList->a[pLevel->iFrom];
6166 if( pSrc->fg.isMaterialized ){
6167 if( pSrc->fg.isCorrelated ){
6168 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
6169 }else{
6170 int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
6171 sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
6172 sqlite3VdbeJumpHere(v, iOnce);
6173 }
6174 }
6175 if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
6176 if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
6177#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
6178 constructAutomaticIndex(pParse, &pWInfo->sWC,
6179 &pTabList->a[pLevel->iFrom], notReady, pLevel);
6180#endif
6181 }else{
6182 sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady);
6183 }
6184 if( db->mallocFailed ) goto whereBeginError;
6185 }
6186 addrExplain = sqlite3WhereExplainOneScan(
6187 pParse, pTabList, pLevel, wctrlFlags
6188 );
6189 pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
6190 notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady);
6191 pWInfo->iContinue = pLevel->addrCont;
6192 if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
6193 sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
6194 }
6195 }
6196
6197 /* Done. */
6198 VdbeModuleComment((v, "Begin WHERE-core"));
6199 pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v);
6200 return pWInfo;
6201
6202 /* Jump here if malloc fails */
6203whereBeginError:
6204 if( pWInfo ){
6205 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
6206 whereInfoFree(db, pWInfo);
6207 }
6208 return 0;
6209}
6210
6211/*
6212** Part of sqlite3WhereEnd() will rewrite opcodes to reference the
6213** index rather than the main table. In SQLITE_DEBUG mode, we want
6214** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine
6215** does that.
6216*/
6217#ifndef SQLITE_DEBUG
6218# define OpcodeRewriteTrace(D,K,P) /* no-op */
6219#else
6220# define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P)
6221 static void sqlite3WhereOpcodeRewriteTrace(
6222 sqlite3 *db,
6223 int pc,
6224 VdbeOp *pOp
6225 ){
6226 if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return;
6227 sqlite3VdbePrintOp(0, pc, pOp);
6228 }
6229#endif
6230
6231#ifdef SQLITE_DEBUG
6232/*
6233** Return true if cursor iCur is opened by instruction k of the
6234** bytecode. Used inside of assert() only.
6235*/
6236static int cursorIsOpen(Vdbe *v, int iCur, int k){
6237 while( k>=0 ){
6238 VdbeOp *pOp = sqlite3VdbeGetOp(v,k--);
6239 if( pOp->p1!=iCur ) continue;
6240 if( pOp->opcode==OP_Close ) return 0;
6241 if( pOp->opcode==OP_OpenRead ) return 1;
6242 if( pOp->opcode==OP_OpenWrite ) return 1;
6243 if( pOp->opcode==OP_OpenDup ) return 1;
6244 if( pOp->opcode==OP_OpenAutoindex ) return 1;
6245 if( pOp->opcode==OP_OpenEphemeral ) return 1;
6246 }
6247 return 0;
6248}
6249#endif /* SQLITE_DEBUG */
6250
6251/*
6252** Generate the end of the WHERE loop. See comments on
6253** sqlite3WhereBegin() for additional information.
6254*/
6255void sqlite3WhereEnd(WhereInfo *pWInfo){
6256 Parse *pParse = pWInfo->pParse;
6257 Vdbe *v = pParse->pVdbe;
6258 int i;
6259 WhereLevel *pLevel;
6260 WhereLoop *pLoop;
6261 SrcList *pTabList = pWInfo->pTabList;
6262 sqlite3 *db = pParse->db;
6263 int iEnd = sqlite3VdbeCurrentAddr(v);
6264 int nRJ = 0;
6265
6266 /* Generate loop termination code.
6267 */
6268 VdbeModuleComment((v, "End WHERE-core"));
6269 for(i=pWInfo->nLevel-1; i>=0; i--){
6270 int addr;
6271 pLevel = &pWInfo->a[i];
6272 if( pLevel->pRJ ){
6273 /* Terminate the subroutine that forms the interior of the loop of
6274 ** the RIGHT JOIN table */
6275 WhereRightJoin *pRJ = pLevel->pRJ;
6276 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
6277 pLevel->addrCont = 0;
6278 pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v);
6279 sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1);
6280 VdbeCoverage(v);
6281 nRJ++;
6282 }
6283 pLoop = pLevel->pWLoop;
6284 if( pLevel->op!=OP_Noop ){
6285#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
6286 int addrSeek = 0;
6287 Index *pIdx;
6288 int n;
6289 if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
6290 && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */
6291 && (pLoop->wsFlags & WHERE_INDEXED)!=0
6292 && (pIdx = pLoop->u.btree.pIndex)->hasStat1
6293 && (n = pLoop->u.btree.nDistinctCol)>0
6294 && pIdx->aiRowLogEst[n]>=36
6295 ){
6296 int r1 = pParse->nMem+1;
6297 int j, op;
6298 for(j=0; j<n; j++){
6299 sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j);
6300 }
6301 pParse->nMem += n+1;
6302 op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT;
6303 addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n);
6304 VdbeCoverageIf(v, op==OP_SeekLT);
6305 VdbeCoverageIf(v, op==OP_SeekGT);
6306 sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
6307 }
6308#endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
6309 /* The common case: Advance to the next row */
6310 if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont);
6311 sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
6312 sqlite3VdbeChangeP5(v, pLevel->p5);
6313 VdbeCoverage(v);
6314 VdbeCoverageIf(v, pLevel->op==OP_Next);
6315 VdbeCoverageIf(v, pLevel->op==OP_Prev);
6316 VdbeCoverageIf(v, pLevel->op==OP_VNext);
6317 if( pLevel->regBignull ){
6318 sqlite3VdbeResolveLabel(v, pLevel->addrBignull);
6319 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1);
6320 VdbeCoverage(v);
6321 }
6322#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
6323 if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
6324#endif
6325 }else if( pLevel->addrCont ){
6326 sqlite3VdbeResolveLabel(v, pLevel->addrCont);
6327 }
6328 if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){
6329 struct InLoop *pIn;
6330 int j;
6331 sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
6332 for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
6333 assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull
6334 || pParse->db->mallocFailed );
6335 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
6336 if( pIn->eEndLoopOp!=OP_Noop ){
6337 if( pIn->nPrefix ){
6338 int bEarlyOut =
6339 (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
6340 && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0;
6341 if( pLevel->iLeftJoin ){
6342 /* For LEFT JOIN queries, cursor pIn->iCur may not have been
6343 ** opened yet. This occurs for WHERE clauses such as
6344 ** "a = ? AND b IN (...)", where the index is on (a, b). If
6345 ** the RHS of the (a=?) is NULL, then the "b IN (...)" may
6346 ** never have been coded, but the body of the loop run to
6347 ** return the null-row. So, if the cursor is not open yet,
6348 ** jump over the OP_Next or OP_Prev instruction about to
6349 ** be coded. */
6350 sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur,
6351 sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut);
6352 VdbeCoverage(v);
6353 }
6354 if( bEarlyOut ){
6355 sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur,
6356 sqlite3VdbeCurrentAddr(v)+2,
6357 pIn->iBase, pIn->nPrefix);
6358 VdbeCoverage(v);
6359 /* Retarget the OP_IsNull against the left operand of IN so
6360 ** it jumps past the OP_IfNoHope. This is because the
6361 ** OP_IsNull also bypasses the OP_Affinity opcode that is
6362 ** required by OP_IfNoHope. */
6363 sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
6364 }
6365 }
6366 sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
6367 VdbeCoverage(v);
6368 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev);
6369 VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next);
6370 }
6371 sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
6372 }
6373 }
6374 sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
6375 if( pLevel->pRJ ){
6376 sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1);
6377 VdbeCoverage(v);
6378 }
6379 if( pLevel->addrSkip ){
6380 sqlite3VdbeGoto(v, pLevel->addrSkip);
6381 VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName));
6382 sqlite3VdbeJumpHere(v, pLevel->addrSkip);
6383 sqlite3VdbeJumpHere(v, pLevel->addrSkip-2);
6384 }
6385#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
6386 if( pLevel->addrLikeRep ){
6387 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
6388 pLevel->addrLikeRep);
6389 VdbeCoverage(v);
6390 }
6391#endif
6392 if( pLevel->iLeftJoin ){
6393 int ws = pLoop->wsFlags;
6394 addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
6395 assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
6396 if( (ws & WHERE_IDX_ONLY)==0 ){
6397 assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor );
6398 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur);
6399 }
6400 if( (ws & WHERE_INDEXED)
6401 || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx)
6402 ){
6403 if( ws & WHERE_MULTI_OR ){
6404 Index *pIx = pLevel->u.pCoveringIdx;
6405 int iDb = sqlite3SchemaToIndex(db, pIx->pSchema);
6406 sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb);
6407 sqlite3VdbeSetP4KeyInfo(pParse, pIx);
6408 }
6409 sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
6410 }
6411 if( pLevel->op==OP_Return ){
6412 sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
6413 }else{
6414 sqlite3VdbeGoto(v, pLevel->addrFirst);
6415 }
6416 sqlite3VdbeJumpHere(v, addr);
6417 }
6418 VdbeModuleComment((v, "End WHERE-loop%d: %s", i,
6419 pWInfo->pTabList->a[pLevel->iFrom].pTab->zName));
6420 }
6421
6422 assert( pWInfo->nLevel<=pTabList->nSrc );
6423 for(i=0, pLevel=pWInfo->a; i<pWInfo->nLevel; i++, pLevel++){
6424 int k, last;
6425 VdbeOp *pOp, *pLastOp;
6426 Index *pIdx = 0;
6427 SrcItem *pTabItem = &pTabList->a[pLevel->iFrom];
6428 Table *pTab = pTabItem->pTab;
6429 assert( pTab!=0 );
6430 pLoop = pLevel->pWLoop;
6431
6432 /* Do RIGHT JOIN processing. Generate code that will output the
6433 ** unmatched rows of the right operand of the RIGHT JOIN with
6434 ** all of the columns of the left operand set to NULL.
6435 */
6436 if( pLevel->pRJ ){
6437 sqlite3WhereRightJoinLoop(pWInfo, i, pLevel);
6438 continue;
6439 }
6440
6441 /* For a co-routine, change all OP_Column references to the table of
6442 ** the co-routine into OP_Copy of result contained in a register.
6443 ** OP_Rowid becomes OP_Null.
6444 */
6445 if( pTabItem->fg.viaCoroutine ){
6446 testcase( pParse->db->mallocFailed );
6447 translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
6448 pTabItem->regResult, 0);
6449 continue;
6450 }
6451
6452 /* If this scan uses an index, make VDBE code substitutions to read data
6453 ** from the index instead of from the table where possible. In some cases
6454 ** this optimization prevents the table from ever being read, which can
6455 ** yield a significant performance boost.
6456 **
6457 ** Calls to the code generator in between sqlite3WhereBegin and
6458 ** sqlite3WhereEnd will have created code that references the table
6459 ** directly. This loop scans all that code looking for opcodes
6460 ** that reference the table and converts them into opcodes that
6461 ** reference the index.
6462 */
6463 if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){
6464 pIdx = pLoop->u.btree.pIndex;
6465 }else if( pLoop->wsFlags & WHERE_MULTI_OR ){
6466 pIdx = pLevel->u.pCoveringIdx;
6467 }
6468 if( pIdx
6469 && !db->mallocFailed
6470 ){
6471 if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){
6472 last = iEnd;
6473 }else{
6474 last = pWInfo->iEndWhere;
6475 }
6476 if( pIdx->bHasExpr ){
6477 IndexedExpr *p = pParse->pIdxExpr;
6478 while( p ){
6479 if( p->iIdxCur==pLevel->iIdxCur ){
6480 p->iDataCur = -1;
6481 p->iIdxCur = -1;
6482 }
6483 p = p->pIENext;
6484 }
6485 }
6486 k = pLevel->addrBody + 1;
6487#ifdef SQLITE_DEBUG
6488 if( db->flags & SQLITE_VdbeAddopTrace ){
6489 printf("TRANSLATE opcodes in range %d..%d\n", k, last-1);
6490 }
6491 /* Proof that the "+1" on the k value above is safe */
6492 pOp = sqlite3VdbeGetOp(v, k - 1);
6493 assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur );
6494 assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur );
6495 assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur );
6496#endif
6497 pOp = sqlite3VdbeGetOp(v, k);
6498 pLastOp = pOp + (last - k);
6499 assert( pOp<=pLastOp );
6500 do{
6501 if( pOp->p1!=pLevel->iTabCur ){
6502 /* no-op */
6503 }else if( pOp->opcode==OP_Column
6504#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
6505 || pOp->opcode==OP_Offset
6506#endif
6507 ){
6508 int x = pOp->p2;
6509 assert( pIdx->pTable==pTab );
6510#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
6511 if( pOp->opcode==OP_Offset ){
6512 /* Do not need to translate the column number */
6513 }else
6514#endif
6515 if( !HasRowid(pTab) ){
6516 Index *pPk = sqlite3PrimaryKeyIndex(pTab);
6517 x = pPk->aiColumn[x];
6518 assert( x>=0 );
6519 }else{
6520 testcase( x!=sqlite3StorageColumnToTable(pTab,x) );
6521 x = sqlite3StorageColumnToTable(pTab,x);
6522 }
6523 x = sqlite3TableColumnToIndex(pIdx, x);
6524 if( x>=0 ){
6525 pOp->p2 = x;
6526 pOp->p1 = pLevel->iIdxCur;
6527 OpcodeRewriteTrace(db, k, pOp);
6528 }else{
6529 /* Unable to translate the table reference into an index
6530 ** reference. Verify that this is harmless - that the
6531 ** table being referenced really is open.
6532 */
6533#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
6534 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
6535 || cursorIsOpen(v,pOp->p1,k)
6536 || pOp->opcode==OP_Offset
6537 );
6538#else
6539 assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
6540 || cursorIsOpen(v,pOp->p1,k)
6541 );
6542#endif
6543 }
6544 }else if( pOp->opcode==OP_Rowid ){
6545 pOp->p1 = pLevel->iIdxCur;
6546 pOp->opcode = OP_IdxRowid;
6547 OpcodeRewriteTrace(db, k, pOp);
6548 }else if( pOp->opcode==OP_IfNullRow ){
6549 pOp->p1 = pLevel->iIdxCur;
6550 OpcodeRewriteTrace(db, k, pOp);
6551 }
6552#ifdef SQLITE_DEBUG
6553 k++;
6554#endif
6555 }while( (++pOp)<pLastOp );
6556#ifdef SQLITE_DEBUG
6557 if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n");
6558#endif
6559 }
6560 }
6561
6562 /* The "break" point is here, just past the end of the outer loop.
6563 ** Set it.
6564 */
6565 sqlite3VdbeResolveLabel(v, pWInfo->iBreak);
6566
6567 /* Final cleanup
6568 */
6569 pParse->nQueryLoop = pWInfo->savedNQueryLoop;
6570 whereInfoFree(db, pWInfo);
6571 pParse->withinRJSubrtn -= nRJ;
6572 return;
6573}
6574