1/*
2** 2001 September 15
3**
4** The author disclaims copyright to this source code. In place of
5** a legal notice, here is a blessing:
6**
7** May you do good and not evil.
8** May you find forgiveness for yourself and forgive others.
9** May you share freely, never taking more than you give.
10**
11*************************************************************************
12** This file contains C code routines that are called by the parser
13** to handle SELECT statements in SQLite.
14*/
15#include "sqliteInt.h"
16
17/*
18** An instance of the following object is used to record information about
19** how to process the DISTINCT keyword, to simplify passing that information
20** into the selectInnerLoop() routine.
21*/
22typedef struct DistinctCtx DistinctCtx;
23struct DistinctCtx {
24 u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */
25 u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */
26 int tabTnct; /* Ephemeral table used for DISTINCT processing */
27 int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */
28};
29
30/*
31** An instance of the following object is used to record information about
32** the ORDER BY (or GROUP BY) clause of query is being coded.
33**
34** The aDefer[] array is used by the sorter-references optimization. For
35** example, assuming there is no index that can be used for the ORDER BY,
36** for the query:
37**
38** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10;
39**
40** it may be more efficient to add just the "a" values to the sorter, and
41** retrieve the associated "bigblob" values directly from table t1 as the
42** 10 smallest "a" values are extracted from the sorter.
43**
44** When the sorter-reference optimization is used, there is one entry in the
45** aDefer[] array for each database table that may be read as values are
46** extracted from the sorter.
47*/
48typedef struct SortCtx SortCtx;
49struct SortCtx {
50 ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */
51 int nOBSat; /* Number of ORDER BY terms satisfied by indices */
52 int iECursor; /* Cursor number for the sorter */
53 int regReturn; /* Register holding block-output return address */
54 int labelBkOut; /* Start label for the block-output subroutine */
55 int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */
56 int labelDone; /* Jump here when done, ex: LIMIT reached */
57 int labelOBLopt; /* Jump here when sorter is full */
58 u8 sortFlags; /* Zero or more SORTFLAG_* bits */
59#ifdef SQLITE_ENABLE_SORTER_REFERENCES
60 u8 nDefer; /* Number of valid entries in aDefer[] */
61 struct DeferredCsr {
62 Table *pTab; /* Table definition */
63 int iCsr; /* Cursor number for table */
64 int nKey; /* Number of PK columns for table pTab (>=1) */
65 } aDefer[4];
66#endif
67 struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */
68};
69#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */
70
71/*
72** Delete all the content of a Select structure. Deallocate the structure
73** itself depending on the value of bFree
74**
75** If bFree==1, call sqlite3DbFree() on the p object.
76** If bFree==0, Leave the first Select object unfreed
77*/
78static void clearSelect(sqlite3 *db, Select *p, int bFree){
79 assert( db!=0 );
80 while( p ){
81 Select *pPrior = p->pPrior;
82 sqlite3ExprListDelete(db, p->pEList);
83 sqlite3SrcListDelete(db, p->pSrc);
84 sqlite3ExprDelete(db, p->pWhere);
85 sqlite3ExprListDelete(db, p->pGroupBy);
86 sqlite3ExprDelete(db, p->pHaving);
87 sqlite3ExprListDelete(db, p->pOrderBy);
88 sqlite3ExprDelete(db, p->pLimit);
89 if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith);
90#ifndef SQLITE_OMIT_WINDOWFUNC
91 if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){
92 sqlite3WindowListDelete(db, p->pWinDefn);
93 }
94 while( p->pWin ){
95 assert( p->pWin->ppThis==&p->pWin );
96 sqlite3WindowUnlinkFromSelect(p->pWin);
97 }
98#endif
99 if( bFree ) sqlite3DbNNFreeNN(db, p);
100 p = pPrior;
101 bFree = 1;
102 }
103}
104
105/*
106** Initialize a SelectDest structure.
107*/
108void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){
109 pDest->eDest = (u8)eDest;
110 pDest->iSDParm = iParm;
111 pDest->iSDParm2 = 0;
112 pDest->zAffSdst = 0;
113 pDest->iSdst = 0;
114 pDest->nSdst = 0;
115}
116
117
118/*
119** Allocate a new Select structure and return a pointer to that
120** structure.
121*/
122Select *sqlite3SelectNew(
123 Parse *pParse, /* Parsing context */
124 ExprList *pEList, /* which columns to include in the result */
125 SrcList *pSrc, /* the FROM clause -- which tables to scan */
126 Expr *pWhere, /* the WHERE clause */
127 ExprList *pGroupBy, /* the GROUP BY clause */
128 Expr *pHaving, /* the HAVING clause */
129 ExprList *pOrderBy, /* the ORDER BY clause */
130 u32 selFlags, /* Flag parameters, such as SF_Distinct */
131 Expr *pLimit /* LIMIT value. NULL means not used */
132){
133 Select *pNew, *pAllocated;
134 Select standin;
135 pAllocated = pNew = sqlite3DbMallocRawNN(pParse->db, sizeof(*pNew) );
136 if( pNew==0 ){
137 assert( pParse->db->mallocFailed );
138 pNew = &standin;
139 }
140 if( pEList==0 ){
141 pEList = sqlite3ExprListAppend(pParse, 0,
142 sqlite3Expr(pParse->db,TK_ASTERISK,0));
143 }
144 pNew->pEList = pEList;
145 pNew->op = TK_SELECT;
146 pNew->selFlags = selFlags;
147 pNew->iLimit = 0;
148 pNew->iOffset = 0;
149 pNew->selId = ++pParse->nSelect;
150 pNew->addrOpenEphm[0] = -1;
151 pNew->addrOpenEphm[1] = -1;
152 pNew->nSelectRow = 0;
153 if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc));
154 pNew->pSrc = pSrc;
155 pNew->pWhere = pWhere;
156 pNew->pGroupBy = pGroupBy;
157 pNew->pHaving = pHaving;
158 pNew->pOrderBy = pOrderBy;
159 pNew->pPrior = 0;
160 pNew->pNext = 0;
161 pNew->pLimit = pLimit;
162 pNew->pWith = 0;
163#ifndef SQLITE_OMIT_WINDOWFUNC
164 pNew->pWin = 0;
165 pNew->pWinDefn = 0;
166#endif
167 if( pParse->db->mallocFailed ) {
168 clearSelect(pParse->db, pNew, pNew!=&standin);
169 pAllocated = 0;
170 }else{
171 assert( pNew->pSrc!=0 || pParse->nErr>0 );
172 }
173 return pAllocated;
174}
175
176
177/*
178** Delete the given Select structure and all of its substructures.
179*/
180void sqlite3SelectDelete(sqlite3 *db, Select *p){
181 if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1);
182}
183
184/*
185** Return a pointer to the right-most SELECT statement in a compound.
186*/
187static Select *findRightmost(Select *p){
188 while( p->pNext ) p = p->pNext;
189 return p;
190}
191
192/*
193** Given 1 to 3 identifiers preceding the JOIN keyword, determine the
194** type of join. Return an integer constant that expresses that type
195** in terms of the following bit values:
196**
197** JT_INNER
198** JT_CROSS
199** JT_OUTER
200** JT_NATURAL
201** JT_LEFT
202** JT_RIGHT
203**
204** A full outer join is the combination of JT_LEFT and JT_RIGHT.
205**
206** If an illegal or unsupported join type is seen, then still return
207** a join type, but put an error in the pParse structure.
208**
209** These are the valid join types:
210**
211**
212** pA pB pC Return Value
213** ------- ----- ----- ------------
214** CROSS - - JT_CROSS
215** INNER - - JT_INNER
216** LEFT - - JT_LEFT|JT_OUTER
217** LEFT OUTER - JT_LEFT|JT_OUTER
218** RIGHT - - JT_RIGHT|JT_OUTER
219** RIGHT OUTER - JT_RIGHT|JT_OUTER
220** FULL - - JT_LEFT|JT_RIGHT|JT_OUTER
221** FULL OUTER - JT_LEFT|JT_RIGHT|JT_OUTER
222** NATURAL INNER - JT_NATURAL|JT_INNER
223** NATURAL LEFT - JT_NATURAL|JT_LEFT|JT_OUTER
224** NATURAL LEFT OUTER JT_NATURAL|JT_LEFT|JT_OUTER
225** NATURAL RIGHT - JT_NATURAL|JT_RIGHT|JT_OUTER
226** NATURAL RIGHT OUTER JT_NATURAL|JT_RIGHT|JT_OUTER
227** NATURAL FULL - JT_NATURAL|JT_LEFT|JT_RIGHT
228** NATURAL FULL OUTER JT_NATRUAL|JT_LEFT|JT_RIGHT
229**
230** To preserve historical compatibly, SQLite also accepts a variety
231** of other non-standard and in many cases non-sensical join types.
232** This routine makes as much sense at it can from the nonsense join
233** type and returns a result. Examples of accepted nonsense join types
234** include but are not limited to:
235**
236** INNER CROSS JOIN -> same as JOIN
237** NATURAL CROSS JOIN -> same as NATURAL JOIN
238** OUTER LEFT JOIN -> same as LEFT JOIN
239** LEFT NATURAL JOIN -> same as NATURAL LEFT JOIN
240** LEFT RIGHT JOIN -> same as FULL JOIN
241** RIGHT OUTER FULL JOIN -> same as FULL JOIN
242** CROSS CROSS CROSS JOIN -> same as JOIN
243**
244** The only restrictions on the join type name are:
245**
246** * "INNER" cannot appear together with "OUTER", "LEFT", "RIGHT",
247** or "FULL".
248**
249** * "CROSS" cannot appear together with "OUTER", "LEFT", "RIGHT,
250** or "FULL".
251**
252** * If "OUTER" is present then there must also be one of
253** "LEFT", "RIGHT", or "FULL"
254*/
255int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){
256 int jointype = 0;
257 Token *apAll[3];
258 Token *p;
259 /* 0123456789 123456789 123456789 123 */
260 static const char zKeyText[] = "naturaleftouterightfullinnercross";
261 static const struct {
262 u8 i; /* Beginning of keyword text in zKeyText[] */
263 u8 nChar; /* Length of the keyword in characters */
264 u8 code; /* Join type mask */
265 } aKeyword[] = {
266 /* (0) natural */ { 0, 7, JT_NATURAL },
267 /* (1) left */ { 6, 4, JT_LEFT|JT_OUTER },
268 /* (2) outer */ { 10, 5, JT_OUTER },
269 /* (3) right */ { 14, 5, JT_RIGHT|JT_OUTER },
270 /* (4) full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER },
271 /* (5) inner */ { 23, 5, JT_INNER },
272 /* (6) cross */ { 28, 5, JT_INNER|JT_CROSS },
273 };
274 int i, j;
275 apAll[0] = pA;
276 apAll[1] = pB;
277 apAll[2] = pC;
278 for(i=0; i<3 && apAll[i]; i++){
279 p = apAll[i];
280 for(j=0; j<ArraySize(aKeyword); j++){
281 if( p->n==aKeyword[j].nChar
282 && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){
283 jointype |= aKeyword[j].code;
284 break;
285 }
286 }
287 testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 );
288 if( j>=ArraySize(aKeyword) ){
289 jointype |= JT_ERROR;
290 break;
291 }
292 }
293 if(
294 (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) ||
295 (jointype & JT_ERROR)!=0 ||
296 (jointype & (JT_OUTER|JT_LEFT|JT_RIGHT))==JT_OUTER
297 ){
298 const char *zSp1 = " ";
299 const char *zSp2 = " ";
300 if( pB==0 ){ zSp1++; }
301 if( pC==0 ){ zSp2++; }
302 sqlite3ErrorMsg(pParse, "unknown join type: "
303 "%T%s%T%s%T", pA, zSp1, pB, zSp2, pC);
304 jointype = JT_INNER;
305 }
306 return jointype;
307}
308
309/*
310** Return the index of a column in a table. Return -1 if the column
311** is not contained in the table.
312*/
313int sqlite3ColumnIndex(Table *pTab, const char *zCol){
314 int i;
315 u8 h = sqlite3StrIHash(zCol);
316 Column *pCol;
317 for(pCol=pTab->aCol, i=0; i<pTab->nCol; pCol++, i++){
318 if( pCol->hName==h && sqlite3StrICmp(pCol->zCnName, zCol)==0 ) return i;
319 }
320 return -1;
321}
322
323/*
324** Mark a subquery result column as having been used.
325*/
326void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){
327 assert( pItem!=0 );
328 assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
329 if( pItem->fg.isNestedFrom ){
330 ExprList *pResults;
331 assert( pItem->pSelect!=0 );
332 pResults = pItem->pSelect->pEList;
333 assert( pResults!=0 );
334 assert( iCol>=0 && iCol<pResults->nExpr );
335 pResults->a[iCol].fg.bUsed = 1;
336 }
337}
338
339/*
340** Search the tables iStart..iEnd (inclusive) in pSrc, looking for a
341** table that has a column named zCol. The search is left-to-right.
342** The first match found is returned.
343**
344** When found, set *piTab and *piCol to the table index and column index
345** of the matching column and return TRUE.
346**
347** If not found, return FALSE.
348*/
349static int tableAndColumnIndex(
350 SrcList *pSrc, /* Array of tables to search */
351 int iStart, /* First member of pSrc->a[] to check */
352 int iEnd, /* Last member of pSrc->a[] to check */
353 const char *zCol, /* Name of the column we are looking for */
354 int *piTab, /* Write index of pSrc->a[] here */
355 int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */
356 int bIgnoreHidden /* Ignore hidden columns */
357){
358 int i; /* For looping over tables in pSrc */
359 int iCol; /* Index of column matching zCol */
360
361 assert( iEnd<pSrc->nSrc );
362 assert( iStart>=0 );
363 assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */
364
365 for(i=iStart; i<=iEnd; i++){
366 iCol = sqlite3ColumnIndex(pSrc->a[i].pTab, zCol);
367 if( iCol>=0
368 && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0)
369 ){
370 if( piTab ){
371 sqlite3SrcItemColumnUsed(&pSrc->a[i], iCol);
372 *piTab = i;
373 *piCol = iCol;
374 }
375 return 1;
376 }
377 }
378 return 0;
379}
380
381/*
382** Set the EP_OuterON property on all terms of the given expression.
383** And set the Expr.w.iJoin to iTable for every term in the
384** expression.
385**
386** The EP_OuterON property is used on terms of an expression to tell
387** the OUTER JOIN processing logic that this term is part of the
388** join restriction specified in the ON or USING clause and not a part
389** of the more general WHERE clause. These terms are moved over to the
390** WHERE clause during join processing but we need to remember that they
391** originated in the ON or USING clause.
392**
393** The Expr.w.iJoin tells the WHERE clause processing that the
394** expression depends on table w.iJoin even if that table is not
395** explicitly mentioned in the expression. That information is needed
396** for cases like this:
397**
398** SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.b AND t1.x=5
399**
400** The where clause needs to defer the handling of the t1.x=5
401** term until after the t2 loop of the join. In that way, a
402** NULL t2 row will be inserted whenever t1.x!=5. If we do not
403** defer the handling of t1.x=5, it will be processed immediately
404** after the t1 loop and rows with t1.x!=5 will never appear in
405** the output, which is incorrect.
406*/
407void sqlite3SetJoinExpr(Expr *p, int iTable, u32 joinFlag){
408 assert( joinFlag==EP_OuterON || joinFlag==EP_InnerON );
409 while( p ){
410 ExprSetProperty(p, joinFlag);
411 assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) );
412 ExprSetVVAProperty(p, EP_NoReduce);
413 p->w.iJoin = iTable;
414 if( p->op==TK_FUNCTION ){
415 assert( ExprUseXList(p) );
416 if( p->x.pList ){
417 int i;
418 for(i=0; i<p->x.pList->nExpr; i++){
419 sqlite3SetJoinExpr(p->x.pList->a[i].pExpr, iTable, joinFlag);
420 }
421 }
422 }
423 sqlite3SetJoinExpr(p->pLeft, iTable, joinFlag);
424 p = p->pRight;
425 }
426}
427
428/* Undo the work of sqlite3SetJoinExpr(). This is used when a LEFT JOIN
429** is simplified into an ordinary JOIN, and when an ON expression is
430** "pushed down" into the WHERE clause of a subquery.
431**
432** Convert every term that is marked with EP_OuterON and w.iJoin==iTable into
433** an ordinary term that omits the EP_OuterON mark. Or if iTable<0, then
434** just clear every EP_OuterON and EP_InnerON mark from the expression tree.
435**
436** If nullable is true, that means that Expr p might evaluate to NULL even
437** if it is a reference to a NOT NULL column. This can happen, for example,
438** if the table that p references is on the left side of a RIGHT JOIN.
439** If nullable is true, then take care to not remove the EP_CanBeNull bit.
440** See forum thread https://sqlite.org/forum/forumpost/b40696f50145d21c
441*/
442static void unsetJoinExpr(Expr *p, int iTable, int nullable){
443 while( p ){
444 if( iTable<0 || (ExprHasProperty(p, EP_OuterON) && p->w.iJoin==iTable) ){
445 ExprClearProperty(p, EP_OuterON|EP_InnerON);
446 if( iTable>=0 ) ExprSetProperty(p, EP_InnerON);
447 }
448 if( p->op==TK_COLUMN && p->iTable==iTable && !nullable ){
449 ExprClearProperty(p, EP_CanBeNull);
450 }
451 if( p->op==TK_FUNCTION ){
452 assert( ExprUseXList(p) );
453 if( p->x.pList ){
454 int i;
455 for(i=0; i<p->x.pList->nExpr; i++){
456 unsetJoinExpr(p->x.pList->a[i].pExpr, iTable, nullable);
457 }
458 }
459 }
460 unsetJoinExpr(p->pLeft, iTable, nullable);
461 p = p->pRight;
462 }
463}
464
465/*
466** This routine processes the join information for a SELECT statement.
467**
468** * A NATURAL join is converted into a USING join. After that, we
469** do not need to be concerned with NATURAL joins and we only have
470** think about USING joins.
471**
472** * ON and USING clauses result in extra terms being added to the
473** WHERE clause to enforce the specified constraints. The extra
474** WHERE clause terms will be tagged with EP_OuterON or
475** EP_InnerON so that we know that they originated in ON/USING.
476**
477** The terms of a FROM clause are contained in the Select.pSrc structure.
478** The left most table is the first entry in Select.pSrc. The right-most
479** table is the last entry. The join operator is held in the entry to
480** the right. Thus entry 1 contains the join operator for the join between
481** entries 0 and 1. Any ON or USING clauses associated with the join are
482** also attached to the right entry.
483**
484** This routine returns the number of errors encountered.
485*/
486static int sqlite3ProcessJoin(Parse *pParse, Select *p){
487 SrcList *pSrc; /* All tables in the FROM clause */
488 int i, j; /* Loop counters */
489 SrcItem *pLeft; /* Left table being joined */
490 SrcItem *pRight; /* Right table being joined */
491
492 pSrc = p->pSrc;
493 pLeft = &pSrc->a[0];
494 pRight = &pLeft[1];
495 for(i=0; i<pSrc->nSrc-1; i++, pRight++, pLeft++){
496 Table *pRightTab = pRight->pTab;
497 u32 joinType;
498
499 if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue;
500 joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON;
501
502 /* If this is a NATURAL join, synthesize an approprate USING clause
503 ** to specify which columns should be joined.
504 */
505 if( pRight->fg.jointype & JT_NATURAL ){
506 IdList *pUsing = 0;
507 if( pRight->fg.isUsing || pRight->u3.pOn ){
508 sqlite3ErrorMsg(pParse, "a NATURAL join may not have "
509 "an ON or USING clause", 0);
510 return 1;
511 }
512 for(j=0; j<pRightTab->nCol; j++){
513 char *zName; /* Name of column in the right table */
514
515 if( IsHiddenColumn(&pRightTab->aCol[j]) ) continue;
516 zName = pRightTab->aCol[j].zCnName;
517 if( tableAndColumnIndex(pSrc, 0, i, zName, 0, 0, 1) ){
518 pUsing = sqlite3IdListAppend(pParse, pUsing, 0);
519 if( pUsing ){
520 assert( pUsing->nId>0 );
521 assert( pUsing->a[pUsing->nId-1].zName==0 );
522 pUsing->a[pUsing->nId-1].zName = sqlite3DbStrDup(pParse->db, zName);
523 }
524 }
525 }
526 if( pUsing ){
527 pRight->fg.isUsing = 1;
528 pRight->fg.isSynthUsing = 1;
529 pRight->u3.pUsing = pUsing;
530 }
531 if( pParse->nErr ) return 1;
532 }
533
534 /* Create extra terms on the WHERE clause for each column named
535 ** in the USING clause. Example: If the two tables to be joined are
536 ** A and B and the USING clause names X, Y, and Z, then add this
537 ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z
538 ** Report an error if any column mentioned in the USING clause is
539 ** not contained in both tables to be joined.
540 */
541 if( pRight->fg.isUsing ){
542 IdList *pList = pRight->u3.pUsing;
543 sqlite3 *db = pParse->db;
544 assert( pList!=0 );
545 for(j=0; j<pList->nId; j++){
546 char *zName; /* Name of the term in the USING clause */
547 int iLeft; /* Table on the left with matching column name */
548 int iLeftCol; /* Column number of matching column on the left */
549 int iRightCol; /* Column number of matching column on the right */
550 Expr *pE1; /* Reference to the column on the LEFT of the join */
551 Expr *pE2; /* Reference to the column on the RIGHT of the join */
552 Expr *pEq; /* Equality constraint. pE1 == pE2 */
553
554 zName = pList->a[j].zName;
555 iRightCol = sqlite3ColumnIndex(pRightTab, zName);
556 if( iRightCol<0
557 || tableAndColumnIndex(pSrc, 0, i, zName, &iLeft, &iLeftCol,
558 pRight->fg.isSynthUsing)==0
559 ){
560 sqlite3ErrorMsg(pParse, "cannot join using column %s - column "
561 "not present in both tables", zName);
562 return 1;
563 }
564 pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol);
565 sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol);
566 if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
567 /* This branch runs if the query contains one or more RIGHT or FULL
568 ** JOINs. If only a single table on the left side of this join
569 ** contains the zName column, then this branch is a no-op.
570 ** But if there are two or more tables on the left side
571 ** of the join, construct a coalesce() function that gathers all
572 ** such tables. Raise an error if more than one of those references
573 ** to zName is not also within a prior USING clause.
574 **
575 ** We really ought to raise an error if there are two or more
576 ** non-USING references to zName on the left of an INNER or LEFT
577 ** JOIN. But older versions of SQLite do not do that, so we avoid
578 ** adding a new error so as to not break legacy applications.
579 */
580 ExprList *pFuncArgs = 0; /* Arguments to the coalesce() */
581 static const Token tkCoalesce = { "coalesce", 8 };
582 while( tableAndColumnIndex(pSrc, iLeft+1, i, zName, &iLeft, &iLeftCol,
583 pRight->fg.isSynthUsing)!=0 ){
584 if( pSrc->a[iLeft].fg.isUsing==0
585 || sqlite3IdListIndex(pSrc->a[iLeft].u3.pUsing, zName)<0
586 ){
587 sqlite3ErrorMsg(pParse, "ambiguous reference to %s in USING()",
588 zName);
589 break;
590 }
591 pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1);
592 pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol);
593 sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol);
594 }
595 if( pFuncArgs ){
596 pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1);
597 pE1 = sqlite3ExprFunction(pParse, pFuncArgs, &tkCoalesce, 0);
598 }
599 }
600 pE2 = sqlite3CreateColumnExpr(db, pSrc, i+1, iRightCol);
601 sqlite3SrcItemColumnUsed(pRight, iRightCol);
602 pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2);
603 assert( pE2!=0 || pEq==0 );
604 if( pEq ){
605 ExprSetProperty(pEq, joinType);
606 assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) );
607 ExprSetVVAProperty(pEq, EP_NoReduce);
608 pEq->w.iJoin = pE2->iTable;
609 }
610 p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pEq);
611 }
612 }
613
614 /* Add the ON clause to the end of the WHERE clause, connected by
615 ** an AND operator.
616 */
617 else if( pRight->u3.pOn ){
618 sqlite3SetJoinExpr(pRight->u3.pOn, pRight->iCursor, joinType);
619 p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->u3.pOn);
620 pRight->u3.pOn = 0;
621 pRight->fg.isOn = 1;
622 }
623 }
624 return 0;
625}
626
627/*
628** An instance of this object holds information (beyond pParse and pSelect)
629** needed to load the next result row that is to be added to the sorter.
630*/
631typedef struct RowLoadInfo RowLoadInfo;
632struct RowLoadInfo {
633 int regResult; /* Store results in array of registers here */
634 u8 ecelFlags; /* Flag argument to ExprCodeExprList() */
635#ifdef SQLITE_ENABLE_SORTER_REFERENCES
636 ExprList *pExtra; /* Extra columns needed by sorter refs */
637 int regExtraResult; /* Where to load the extra columns */
638#endif
639};
640
641/*
642** This routine does the work of loading query data into an array of
643** registers so that it can be added to the sorter.
644*/
645static void innerLoopLoadRow(
646 Parse *pParse, /* Statement under construction */
647 Select *pSelect, /* The query being coded */
648 RowLoadInfo *pInfo /* Info needed to complete the row load */
649){
650 sqlite3ExprCodeExprList(pParse, pSelect->pEList, pInfo->regResult,
651 0, pInfo->ecelFlags);
652#ifdef SQLITE_ENABLE_SORTER_REFERENCES
653 if( pInfo->pExtra ){
654 sqlite3ExprCodeExprList(pParse, pInfo->pExtra, pInfo->regExtraResult, 0, 0);
655 sqlite3ExprListDelete(pParse->db, pInfo->pExtra);
656 }
657#endif
658}
659
660/*
661** Code the OP_MakeRecord instruction that generates the entry to be
662** added into the sorter.
663**
664** Return the register in which the result is stored.
665*/
666static int makeSorterRecord(
667 Parse *pParse,
668 SortCtx *pSort,
669 Select *pSelect,
670 int regBase,
671 int nBase
672){
673 int nOBSat = pSort->nOBSat;
674 Vdbe *v = pParse->pVdbe;
675 int regOut = ++pParse->nMem;
676 if( pSort->pDeferredRowLoad ){
677 innerLoopLoadRow(pParse, pSelect, pSort->pDeferredRowLoad);
678 }
679 sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regOut);
680 return regOut;
681}
682
683/*
684** Generate code that will push the record in registers regData
685** through regData+nData-1 onto the sorter.
686*/
687static void pushOntoSorter(
688 Parse *pParse, /* Parser context */
689 SortCtx *pSort, /* Information about the ORDER BY clause */
690 Select *pSelect, /* The whole SELECT statement */
691 int regData, /* First register holding data to be sorted */
692 int regOrigData, /* First register holding data before packing */
693 int nData, /* Number of elements in the regData data array */
694 int nPrefixReg /* No. of reg prior to regData available for use */
695){
696 Vdbe *v = pParse->pVdbe; /* Stmt under construction */
697 int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0);
698 int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */
699 int nBase = nExpr + bSeq + nData; /* Fields in sorter record */
700 int regBase; /* Regs for sorter record */
701 int regRecord = 0; /* Assembled sorter record */
702 int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */
703 int op; /* Opcode to add sorter record to sorter */
704 int iLimit; /* LIMIT counter */
705 int iSkip = 0; /* End of the sorter insert loop */
706
707 assert( bSeq==0 || bSeq==1 );
708
709 /* Three cases:
710 ** (1) The data to be sorted has already been packed into a Record
711 ** by a prior OP_MakeRecord. In this case nData==1 and regData
712 ** will be completely unrelated to regOrigData.
713 ** (2) All output columns are included in the sort record. In that
714 ** case regData==regOrigData.
715 ** (3) Some output columns are omitted from the sort record due to
716 ** the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the
717 ** SQLITE_ECEL_OMITREF optimization, or due to the
718 ** SortCtx.pDeferredRowLoad optimiation. In any of these cases
719 ** regOrigData is 0 to prevent this routine from trying to copy
720 ** values that might not yet exist.
721 */
722 assert( nData==1 || regData==regOrigData || regOrigData==0 );
723
724 if( nPrefixReg ){
725 assert( nPrefixReg==nExpr+bSeq );
726 regBase = regData - nPrefixReg;
727 }else{
728 regBase = pParse->nMem + 1;
729 pParse->nMem += nBase;
730 }
731 assert( pSelect->iOffset==0 || pSelect->iLimit!=0 );
732 iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit;
733 pSort->labelDone = sqlite3VdbeMakeLabel(pParse);
734 sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData,
735 SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0));
736 if( bSeq ){
737 sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr);
738 }
739 if( nPrefixReg==0 && nData>0 ){
740 sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData);
741 }
742 if( nOBSat>0 ){
743 int regPrevKey; /* The first nOBSat columns of the previous row */
744 int addrFirst; /* Address of the OP_IfNot opcode */
745 int addrJmp; /* Address of the OP_Jump opcode */
746 VdbeOp *pOp; /* Opcode that opens the sorter */
747 int nKey; /* Number of sorting key columns, including OP_Sequence */
748 KeyInfo *pKI; /* Original KeyInfo on the sorter table */
749
750 regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase);
751 regPrevKey = pParse->nMem+1;
752 pParse->nMem += pSort->nOBSat;
753 nKey = nExpr - pSort->nOBSat + bSeq;
754 if( bSeq ){
755 addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr);
756 }else{
757 addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor);
758 }
759 VdbeCoverage(v);
760 sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat);
761 pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex);
762 if( pParse->db->mallocFailed ) return;
763 pOp->p2 = nKey + nData;
764 pKI = pOp->p4.pKeyInfo;
765 memset(pKI->aSortFlags, 0, pKI->nKeyField); /* Makes OP_Jump testable */
766 sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO);
767 testcase( pKI->nAllField > pKI->nKeyField+2 );
768 pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat,
769 pKI->nAllField-pKI->nKeyField-1);
770 pOp = 0; /* Ensure pOp not used after sqltie3VdbeAddOp3() */
771 addrJmp = sqlite3VdbeCurrentAddr(v);
772 sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v);
773 pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse);
774 pSort->regReturn = ++pParse->nMem;
775 sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
776 sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor);
777 if( iLimit ){
778 sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone);
779 VdbeCoverage(v);
780 }
781 sqlite3VdbeJumpHere(v, addrFirst);
782 sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat);
783 sqlite3VdbeJumpHere(v, addrJmp);
784 }
785 if( iLimit ){
786 /* At this point the values for the new sorter entry are stored
787 ** in an array of registers. They need to be composed into a record
788 ** and inserted into the sorter if either (a) there are currently
789 ** less than LIMIT+OFFSET items or (b) the new record is smaller than
790 ** the largest record currently in the sorter. If (b) is true and there
791 ** are already LIMIT+OFFSET items in the sorter, delete the largest
792 ** entry before inserting the new one. This way there are never more
793 ** than LIMIT+OFFSET items in the sorter.
794 **
795 ** If the new record does not need to be inserted into the sorter,
796 ** jump to the next iteration of the loop. If the pSort->labelOBLopt
797 ** value is not zero, then it is a label of where to jump. Otherwise,
798 ** just bypass the row insert logic. See the header comment on the
799 ** sqlite3WhereOrderByLimitOptLabel() function for additional info.
800 */
801 int iCsr = pSort->iECursor;
802 sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4);
803 VdbeCoverage(v);
804 sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0);
805 iSkip = sqlite3VdbeAddOp4Int(v, OP_IdxLE,
806 iCsr, 0, regBase+nOBSat, nExpr-nOBSat);
807 VdbeCoverage(v);
808 sqlite3VdbeAddOp1(v, OP_Delete, iCsr);
809 }
810 if( regRecord==0 ){
811 regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase);
812 }
813 if( pSort->sortFlags & SORTFLAG_UseSorter ){
814 op = OP_SorterInsert;
815 }else{
816 op = OP_IdxInsert;
817 }
818 sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord,
819 regBase+nOBSat, nBase-nOBSat);
820 if( iSkip ){
821 sqlite3VdbeChangeP2(v, iSkip,
822 pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v));
823 }
824}
825
826/*
827** Add code to implement the OFFSET
828*/
829static void codeOffset(
830 Vdbe *v, /* Generate code into this VM */
831 int iOffset, /* Register holding the offset counter */
832 int iContinue /* Jump here to skip the current record */
833){
834 if( iOffset>0 ){
835 sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v);
836 VdbeComment((v, "OFFSET"));
837 }
838}
839
840/*
841** Add code that will check to make sure the array of registers starting at
842** iMem form a distinct entry. This is used by both "SELECT DISTINCT ..." and
843** distinct aggregates ("SELECT count(DISTINCT <expr>) ..."). Three strategies
844** are available. Which is used depends on the value of parameter eTnctType,
845** as follows:
846**
847** WHERE_DISTINCT_UNORDERED/WHERE_DISTINCT_NOOP:
848** Build an ephemeral table that contains all entries seen before and
849** skip entries which have been seen before.
850**
851** Parameter iTab is the cursor number of an ephemeral table that must
852** be opened before the VM code generated by this routine is executed.
853** The ephemeral cursor table is queried for a record identical to the
854** record formed by the current array of registers. If one is found,
855** jump to VM address addrRepeat. Otherwise, insert a new record into
856** the ephemeral cursor and proceed.
857**
858** The returned value in this case is a copy of parameter iTab.
859**
860** WHERE_DISTINCT_ORDERED:
861** In this case rows are being delivered sorted order. The ephermal
862** table is not required. Instead, the current set of values
863** is compared against previous row. If they match, the new row
864** is not distinct and control jumps to VM address addrRepeat. Otherwise,
865** the VM program proceeds with processing the new row.
866**
867** The returned value in this case is the register number of the first
868** in an array of registers used to store the previous result row so that
869** it can be compared to the next. The caller must ensure that this
870** register is initialized to NULL. (The fixDistinctOpenEph() routine
871** will take care of this initialization.)
872**
873** WHERE_DISTINCT_UNIQUE:
874** In this case it has already been determined that the rows are distinct.
875** No special action is required. The return value is zero.
876**
877** Parameter pEList is the list of expressions used to generated the
878** contents of each row. It is used by this routine to determine (a)
879** how many elements there are in the array of registers and (b) the
880** collation sequences that should be used for the comparisons if
881** eTnctType is WHERE_DISTINCT_ORDERED.
882*/
883static int codeDistinct(
884 Parse *pParse, /* Parsing and code generating context */
885 int eTnctType, /* WHERE_DISTINCT_* value */
886 int iTab, /* A sorting index used to test for distinctness */
887 int addrRepeat, /* Jump to here if not distinct */
888 ExprList *pEList, /* Expression for each element */
889 int regElem /* First element */
890){
891 int iRet = 0;
892 int nResultCol = pEList->nExpr;
893 Vdbe *v = pParse->pVdbe;
894
895 switch( eTnctType ){
896 case WHERE_DISTINCT_ORDERED: {
897 int i;
898 int iJump; /* Jump destination */
899 int regPrev; /* Previous row content */
900
901 /* Allocate space for the previous row */
902 iRet = regPrev = pParse->nMem+1;
903 pParse->nMem += nResultCol;
904
905 iJump = sqlite3VdbeCurrentAddr(v) + nResultCol;
906 for(i=0; i<nResultCol; i++){
907 CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[i].pExpr);
908 if( i<nResultCol-1 ){
909 sqlite3VdbeAddOp3(v, OP_Ne, regElem+i, iJump, regPrev+i);
910 VdbeCoverage(v);
911 }else{
912 sqlite3VdbeAddOp3(v, OP_Eq, regElem+i, addrRepeat, regPrev+i);
913 VdbeCoverage(v);
914 }
915 sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
916 sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
917 }
918 assert( sqlite3VdbeCurrentAddr(v)==iJump || pParse->db->mallocFailed );
919 sqlite3VdbeAddOp3(v, OP_Copy, regElem, regPrev, nResultCol-1);
920 break;
921 }
922
923 case WHERE_DISTINCT_UNIQUE: {
924 /* nothing to do */
925 break;
926 }
927
928 default: {
929 int r1 = sqlite3GetTempReg(pParse);
930 sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, regElem, nResultCol);
931 VdbeCoverage(v);
932 sqlite3VdbeAddOp3(v, OP_MakeRecord, regElem, nResultCol, r1);
933 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol);
934 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
935 sqlite3ReleaseTempReg(pParse, r1);
936 iRet = iTab;
937 break;
938 }
939 }
940
941 return iRet;
942}
943
944/*
945** This routine runs after codeDistinct(). It makes necessary
946** adjustments to the OP_OpenEphemeral opcode that the codeDistinct()
947** routine made use of. This processing must be done separately since
948** sometimes codeDistinct is called before the OP_OpenEphemeral is actually
949** laid down.
950**
951** WHERE_DISTINCT_NOOP:
952** WHERE_DISTINCT_UNORDERED:
953**
954** No adjustments necessary. This function is a no-op.
955**
956** WHERE_DISTINCT_UNIQUE:
957**
958** The ephemeral table is not needed. So change the
959** OP_OpenEphemeral opcode into an OP_Noop.
960**
961** WHERE_DISTINCT_ORDERED:
962**
963** The ephemeral table is not needed. But we do need register
964** iVal to be initialized to NULL. So change the OP_OpenEphemeral
965** into an OP_Null on the iVal register.
966*/
967static void fixDistinctOpenEph(
968 Parse *pParse, /* Parsing and code generating context */
969 int eTnctType, /* WHERE_DISTINCT_* value */
970 int iVal, /* Value returned by codeDistinct() */
971 int iOpenEphAddr /* Address of OP_OpenEphemeral instruction for iTab */
972){
973 if( pParse->nErr==0
974 && (eTnctType==WHERE_DISTINCT_UNIQUE || eTnctType==WHERE_DISTINCT_ORDERED)
975 ){
976 Vdbe *v = pParse->pVdbe;
977 sqlite3VdbeChangeToNoop(v, iOpenEphAddr);
978 if( sqlite3VdbeGetOp(v, iOpenEphAddr+1)->opcode==OP_Explain ){
979 sqlite3VdbeChangeToNoop(v, iOpenEphAddr+1);
980 }
981 if( eTnctType==WHERE_DISTINCT_ORDERED ){
982 /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared
983 ** bit on the first register of the previous value. This will cause the
984 ** OP_Ne added in codeDistinct() to always fail on the first iteration of
985 ** the loop even if the first row is all NULLs. */
986 VdbeOp *pOp = sqlite3VdbeGetOp(v, iOpenEphAddr);
987 pOp->opcode = OP_Null;
988 pOp->p1 = 1;
989 pOp->p2 = iVal;
990 }
991 }
992}
993
994#ifdef SQLITE_ENABLE_SORTER_REFERENCES
995/*
996** This function is called as part of inner-loop generation for a SELECT
997** statement with an ORDER BY that is not optimized by an index. It
998** determines the expressions, if any, that the sorter-reference
999** optimization should be used for. The sorter-reference optimization
1000** is used for SELECT queries like:
1001**
1002** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10
1003**
1004** If the optimization is used for expression "bigblob", then instead of
1005** storing values read from that column in the sorter records, the PK of
1006** the row from table t1 is stored instead. Then, as records are extracted from
1007** the sorter to return to the user, the required value of bigblob is
1008** retrieved directly from table t1. If the values are very large, this
1009** can be more efficient than storing them directly in the sorter records.
1010**
1011** The ExprList_item.fg.bSorterRef flag is set for each expression in pEList
1012** for which the sorter-reference optimization should be enabled.
1013** Additionally, the pSort->aDefer[] array is populated with entries
1014** for all cursors required to evaluate all selected expressions. Finally.
1015** output variable (*ppExtra) is set to an expression list containing
1016** expressions for all extra PK values that should be stored in the
1017** sorter records.
1018*/
1019static void selectExprDefer(
1020 Parse *pParse, /* Leave any error here */
1021 SortCtx *pSort, /* Sorter context */
1022 ExprList *pEList, /* Expressions destined for sorter */
1023 ExprList **ppExtra /* Expressions to append to sorter record */
1024){
1025 int i;
1026 int nDefer = 0;
1027 ExprList *pExtra = 0;
1028 for(i=0; i<pEList->nExpr; i++){
1029 struct ExprList_item *pItem = &pEList->a[i];
1030 if( pItem->u.x.iOrderByCol==0 ){
1031 Expr *pExpr = pItem->pExpr;
1032 Table *pTab;
1033 if( pExpr->op==TK_COLUMN
1034 && pExpr->iColumn>=0
1035 && ALWAYS( ExprUseYTab(pExpr) )
1036 && (pTab = pExpr->y.pTab)!=0
1037 && IsOrdinaryTable(pTab)
1038 && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF)!=0
1039 ){
1040 int j;
1041 for(j=0; j<nDefer; j++){
1042 if( pSort->aDefer[j].iCsr==pExpr->iTable ) break;
1043 }
1044 if( j==nDefer ){
1045 if( nDefer==ArraySize(pSort->aDefer) ){
1046 continue;
1047 }else{
1048 int nKey = 1;
1049 int k;
1050 Index *pPk = 0;
1051 if( !HasRowid(pTab) ){
1052 pPk = sqlite3PrimaryKeyIndex(pTab);
1053 nKey = pPk->nKeyCol;
1054 }
1055 for(k=0; k<nKey; k++){
1056 Expr *pNew = sqlite3PExpr(pParse, TK_COLUMN, 0, 0);
1057 if( pNew ){
1058 pNew->iTable = pExpr->iTable;
1059 assert( ExprUseYTab(pNew) );
1060 pNew->y.pTab = pExpr->y.pTab;
1061 pNew->iColumn = pPk ? pPk->aiColumn[k] : -1;
1062 pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew);
1063 }
1064 }
1065 pSort->aDefer[nDefer].pTab = pExpr->y.pTab;
1066 pSort->aDefer[nDefer].iCsr = pExpr->iTable;
1067 pSort->aDefer[nDefer].nKey = nKey;
1068 nDefer++;
1069 }
1070 }
1071 pItem->fg.bSorterRef = 1;
1072 }
1073 }
1074 }
1075 pSort->nDefer = (u8)nDefer;
1076 *ppExtra = pExtra;
1077}
1078#endif
1079
1080/*
1081** This routine generates the code for the inside of the inner loop
1082** of a SELECT.
1083**
1084** If srcTab is negative, then the p->pEList expressions
1085** are evaluated in order to get the data for this row. If srcTab is
1086** zero or more, then data is pulled from srcTab and p->pEList is used only
1087** to get the number of columns and the collation sequence for each column.
1088*/
1089static void selectInnerLoop(
1090 Parse *pParse, /* The parser context */
1091 Select *p, /* The complete select statement being coded */
1092 int srcTab, /* Pull data from this table if non-negative */
1093 SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */
1094 DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */
1095 SelectDest *pDest, /* How to dispose of the results */
1096 int iContinue, /* Jump here to continue with next row */
1097 int iBreak /* Jump here to break out of the inner loop */
1098){
1099 Vdbe *v = pParse->pVdbe;
1100 int i;
1101 int hasDistinct; /* True if the DISTINCT keyword is present */
1102 int eDest = pDest->eDest; /* How to dispose of results */
1103 int iParm = pDest->iSDParm; /* First argument to disposal method */
1104 int nResultCol; /* Number of result columns */
1105 int nPrefixReg = 0; /* Number of extra registers before regResult */
1106 RowLoadInfo sRowLoadInfo; /* Info for deferred row loading */
1107
1108 /* Usually, regResult is the first cell in an array of memory cells
1109 ** containing the current result row. In this case regOrig is set to the
1110 ** same value. However, if the results are being sent to the sorter, the
1111 ** values for any expressions that are also part of the sort-key are omitted
1112 ** from this array. In this case regOrig is set to zero. */
1113 int regResult; /* Start of memory holding current results */
1114 int regOrig; /* Start of memory holding full result (or 0) */
1115
1116 assert( v );
1117 assert( p->pEList!=0 );
1118 hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP;
1119 if( pSort && pSort->pOrderBy==0 ) pSort = 0;
1120 if( pSort==0 && !hasDistinct ){
1121 assert( iContinue!=0 );
1122 codeOffset(v, p->iOffset, iContinue);
1123 }
1124
1125 /* Pull the requested columns.
1126 */
1127 nResultCol = p->pEList->nExpr;
1128
1129 if( pDest->iSdst==0 ){
1130 if( pSort ){
1131 nPrefixReg = pSort->pOrderBy->nExpr;
1132 if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++;
1133 pParse->nMem += nPrefixReg;
1134 }
1135 pDest->iSdst = pParse->nMem+1;
1136 pParse->nMem += nResultCol;
1137 }else if( pDest->iSdst+nResultCol > pParse->nMem ){
1138 /* This is an error condition that can result, for example, when a SELECT
1139 ** on the right-hand side of an INSERT contains more result columns than
1140 ** there are columns in the table on the left. The error will be caught
1141 ** and reported later. But we need to make sure enough memory is allocated
1142 ** to avoid other spurious errors in the meantime. */
1143 pParse->nMem += nResultCol;
1144 }
1145 pDest->nSdst = nResultCol;
1146 regOrig = regResult = pDest->iSdst;
1147 if( srcTab>=0 ){
1148 for(i=0; i<nResultCol; i++){
1149 sqlite3VdbeAddOp3(v, OP_Column, srcTab, i, regResult+i);
1150 VdbeComment((v, "%s", p->pEList->a[i].zEName));
1151 }
1152 }else if( eDest!=SRT_Exists ){
1153#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1154 ExprList *pExtra = 0;
1155#endif
1156 /* If the destination is an EXISTS(...) expression, the actual
1157 ** values returned by the SELECT are not required.
1158 */
1159 u8 ecelFlags; /* "ecel" is an abbreviation of "ExprCodeExprList" */
1160 ExprList *pEList;
1161 if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){
1162 ecelFlags = SQLITE_ECEL_DUP;
1163 }else{
1164 ecelFlags = 0;
1165 }
1166 if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){
1167 /* For each expression in p->pEList that is a copy of an expression in
1168 ** the ORDER BY clause (pSort->pOrderBy), set the associated
1169 ** iOrderByCol value to one more than the index of the ORDER BY
1170 ** expression within the sort-key that pushOntoSorter() will generate.
1171 ** This allows the p->pEList field to be omitted from the sorted record,
1172 ** saving space and CPU cycles. */
1173 ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF);
1174
1175 for(i=pSort->nOBSat; i<pSort->pOrderBy->nExpr; i++){
1176 int j;
1177 if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){
1178 p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat;
1179 }
1180 }
1181#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1182 selectExprDefer(pParse, pSort, p->pEList, &pExtra);
1183 if( pExtra && pParse->db->mallocFailed==0 ){
1184 /* If there are any extra PK columns to add to the sorter records,
1185 ** allocate extra memory cells and adjust the OpenEphemeral
1186 ** instruction to account for the larger records. This is only
1187 ** required if there are one or more WITHOUT ROWID tables with
1188 ** composite primary keys in the SortCtx.aDefer[] array. */
1189 VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex);
1190 pOp->p2 += (pExtra->nExpr - pSort->nDefer);
1191 pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer);
1192 pParse->nMem += pExtra->nExpr;
1193 }
1194#endif
1195
1196 /* Adjust nResultCol to account for columns that are omitted
1197 ** from the sorter by the optimizations in this branch */
1198 pEList = p->pEList;
1199 for(i=0; i<pEList->nExpr; i++){
1200 if( pEList->a[i].u.x.iOrderByCol>0
1201#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1202 || pEList->a[i].fg.bSorterRef
1203#endif
1204 ){
1205 nResultCol--;
1206 regOrig = 0;
1207 }
1208 }
1209
1210 testcase( regOrig );
1211 testcase( eDest==SRT_Set );
1212 testcase( eDest==SRT_Mem );
1213 testcase( eDest==SRT_Coroutine );
1214 testcase( eDest==SRT_Output );
1215 assert( eDest==SRT_Set || eDest==SRT_Mem
1216 || eDest==SRT_Coroutine || eDest==SRT_Output
1217 || eDest==SRT_Upfrom );
1218 }
1219 sRowLoadInfo.regResult = regResult;
1220 sRowLoadInfo.ecelFlags = ecelFlags;
1221#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1222 sRowLoadInfo.pExtra = pExtra;
1223 sRowLoadInfo.regExtraResult = regResult + nResultCol;
1224 if( pExtra ) nResultCol += pExtra->nExpr;
1225#endif
1226 if( p->iLimit
1227 && (ecelFlags & SQLITE_ECEL_OMITREF)!=0
1228 && nPrefixReg>0
1229 ){
1230 assert( pSort!=0 );
1231 assert( hasDistinct==0 );
1232 pSort->pDeferredRowLoad = &sRowLoadInfo;
1233 regOrig = 0;
1234 }else{
1235 innerLoopLoadRow(pParse, p, &sRowLoadInfo);
1236 }
1237 }
1238
1239 /* If the DISTINCT keyword was present on the SELECT statement
1240 ** and this row has been seen before, then do not make this row
1241 ** part of the result.
1242 */
1243 if( hasDistinct ){
1244 int eType = pDistinct->eTnctType;
1245 int iTab = pDistinct->tabTnct;
1246 assert( nResultCol==p->pEList->nExpr );
1247 iTab = codeDistinct(pParse, eType, iTab, iContinue, p->pEList, regResult);
1248 fixDistinctOpenEph(pParse, eType, iTab, pDistinct->addrTnct);
1249 if( pSort==0 ){
1250 codeOffset(v, p->iOffset, iContinue);
1251 }
1252 }
1253
1254 switch( eDest ){
1255 /* In this mode, write each query result to the key of the temporary
1256 ** table iParm.
1257 */
1258#ifndef SQLITE_OMIT_COMPOUND_SELECT
1259 case SRT_Union: {
1260 int r1;
1261 r1 = sqlite3GetTempReg(pParse);
1262 sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1);
1263 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol);
1264 sqlite3ReleaseTempReg(pParse, r1);
1265 break;
1266 }
1267
1268 /* Construct a record from the query result, but instead of
1269 ** saving that record, use it as a key to delete elements from
1270 ** the temporary table iParm.
1271 */
1272 case SRT_Except: {
1273 sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol);
1274 break;
1275 }
1276#endif /* SQLITE_OMIT_COMPOUND_SELECT */
1277
1278 /* Store the result as data using a unique key.
1279 */
1280 case SRT_Fifo:
1281 case SRT_DistFifo:
1282 case SRT_Table:
1283 case SRT_EphemTab: {
1284 int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1);
1285 testcase( eDest==SRT_Table );
1286 testcase( eDest==SRT_EphemTab );
1287 testcase( eDest==SRT_Fifo );
1288 testcase( eDest==SRT_DistFifo );
1289 sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg);
1290 if( pDest->zAffSdst ){
1291 sqlite3VdbeChangeP4(v, -1, pDest->zAffSdst, nResultCol);
1292 }
1293#ifndef SQLITE_OMIT_CTE
1294 if( eDest==SRT_DistFifo ){
1295 /* If the destination is DistFifo, then cursor (iParm+1) is open
1296 ** on an ephemeral index. If the current row is already present
1297 ** in the index, do not write it to the output. If not, add the
1298 ** current row to the index and proceed with writing it to the
1299 ** output table as well. */
1300 int addr = sqlite3VdbeCurrentAddr(v) + 4;
1301 sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0);
1302 VdbeCoverage(v);
1303 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1,regResult,nResultCol);
1304 assert( pSort==0 );
1305 }
1306#endif
1307 if( pSort ){
1308 assert( regResult==regOrig );
1309 pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, regOrig, 1, nPrefixReg);
1310 }else{
1311 int r2 = sqlite3GetTempReg(pParse);
1312 sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2);
1313 sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2);
1314 sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
1315 sqlite3ReleaseTempReg(pParse, r2);
1316 }
1317 sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1);
1318 break;
1319 }
1320
1321 case SRT_Upfrom: {
1322 if( pSort ){
1323 pushOntoSorter(
1324 pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg);
1325 }else{
1326 int i2 = pDest->iSDParm2;
1327 int r1 = sqlite3GetTempReg(pParse);
1328
1329 /* If the UPDATE FROM join is an aggregate that matches no rows, it
1330 ** might still be trying to return one row, because that is what
1331 ** aggregates do. Don't record that empty row in the output table. */
1332 sqlite3VdbeAddOp2(v, OP_IsNull, regResult, iBreak); VdbeCoverage(v);
1333
1334 sqlite3VdbeAddOp3(v, OP_MakeRecord,
1335 regResult+(i2<0), nResultCol-(i2<0), r1);
1336 if( i2<0 ){
1337 sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regResult);
1338 }else{
1339 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, i2);
1340 }
1341 }
1342 break;
1343 }
1344
1345#ifndef SQLITE_OMIT_SUBQUERY
1346 /* If we are creating a set for an "expr IN (SELECT ...)" construct,
1347 ** then there should be a single item on the stack. Write this
1348 ** item into the set table with bogus data.
1349 */
1350 case SRT_Set: {
1351 if( pSort ){
1352 /* At first glance you would think we could optimize out the
1353 ** ORDER BY in this case since the order of entries in the set
1354 ** does not matter. But there might be a LIMIT clause, in which
1355 ** case the order does matter */
1356 pushOntoSorter(
1357 pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg);
1358 }else{
1359 int r1 = sqlite3GetTempReg(pParse);
1360 assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol );
1361 sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol,
1362 r1, pDest->zAffSdst, nResultCol);
1363 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol);
1364 sqlite3ReleaseTempReg(pParse, r1);
1365 }
1366 break;
1367 }
1368
1369
1370 /* If any row exist in the result set, record that fact and abort.
1371 */
1372 case SRT_Exists: {
1373 sqlite3VdbeAddOp2(v, OP_Integer, 1, iParm);
1374 /* The LIMIT clause will terminate the loop for us */
1375 break;
1376 }
1377
1378 /* If this is a scalar select that is part of an expression, then
1379 ** store the results in the appropriate memory cell or array of
1380 ** memory cells and break out of the scan loop.
1381 */
1382 case SRT_Mem: {
1383 if( pSort ){
1384 assert( nResultCol<=pDest->nSdst );
1385 pushOntoSorter(
1386 pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg);
1387 }else{
1388 assert( nResultCol==pDest->nSdst );
1389 assert( regResult==iParm );
1390 /* The LIMIT clause will jump out of the loop for us */
1391 }
1392 break;
1393 }
1394#endif /* #ifndef SQLITE_OMIT_SUBQUERY */
1395
1396 case SRT_Coroutine: /* Send data to a co-routine */
1397 case SRT_Output: { /* Return the results */
1398 testcase( eDest==SRT_Coroutine );
1399 testcase( eDest==SRT_Output );
1400 if( pSort ){
1401 pushOntoSorter(pParse, pSort, p, regResult, regOrig, nResultCol,
1402 nPrefixReg);
1403 }else if( eDest==SRT_Coroutine ){
1404 sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm);
1405 }else{
1406 sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol);
1407 }
1408 break;
1409 }
1410
1411#ifndef SQLITE_OMIT_CTE
1412 /* Write the results into a priority queue that is order according to
1413 ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an
1414 ** index with pSO->nExpr+2 columns. Build a key using pSO for the first
1415 ** pSO->nExpr columns, then make sure all keys are unique by adding a
1416 ** final OP_Sequence column. The last column is the record as a blob.
1417 */
1418 case SRT_DistQueue:
1419 case SRT_Queue: {
1420 int nKey;
1421 int r1, r2, r3;
1422 int addrTest = 0;
1423 ExprList *pSO;
1424 pSO = pDest->pOrderBy;
1425 assert( pSO );
1426 nKey = pSO->nExpr;
1427 r1 = sqlite3GetTempReg(pParse);
1428 r2 = sqlite3GetTempRange(pParse, nKey+2);
1429 r3 = r2+nKey+1;
1430 if( eDest==SRT_DistQueue ){
1431 /* If the destination is DistQueue, then cursor (iParm+1) is open
1432 ** on a second ephemeral index that holds all values every previously
1433 ** added to the queue. */
1434 addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0,
1435 regResult, nResultCol);
1436 VdbeCoverage(v);
1437 }
1438 sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3);
1439 if( eDest==SRT_DistQueue ){
1440 sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3);
1441 sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
1442 }
1443 for(i=0; i<nKey; i++){
1444 sqlite3VdbeAddOp2(v, OP_SCopy,
1445 regResult + pSO->a[i].u.x.iOrderByCol - 1,
1446 r2+i);
1447 }
1448 sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey);
1449 sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1);
1450 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2);
1451 if( addrTest ) sqlite3VdbeJumpHere(v, addrTest);
1452 sqlite3ReleaseTempReg(pParse, r1);
1453 sqlite3ReleaseTempRange(pParse, r2, nKey+2);
1454 break;
1455 }
1456#endif /* SQLITE_OMIT_CTE */
1457
1458
1459
1460#if !defined(SQLITE_OMIT_TRIGGER)
1461 /* Discard the results. This is used for SELECT statements inside
1462 ** the body of a TRIGGER. The purpose of such selects is to call
1463 ** user-defined functions that have side effects. We do not care
1464 ** about the actual results of the select.
1465 */
1466 default: {
1467 assert( eDest==SRT_Discard );
1468 break;
1469 }
1470#endif
1471 }
1472
1473 /* Jump to the end of the loop if the LIMIT is reached. Except, if
1474 ** there is a sorter, in which case the sorter has already limited
1475 ** the output for us.
1476 */
1477 if( pSort==0 && p->iLimit ){
1478 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v);
1479 }
1480}
1481
1482/*
1483** Allocate a KeyInfo object sufficient for an index of N key columns and
1484** X extra columns.
1485*/
1486KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){
1487 int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*);
1488 KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra);
1489 if( p ){
1490 p->aSortFlags = (u8*)&p->aColl[N+X];
1491 p->nKeyField = (u16)N;
1492 p->nAllField = (u16)(N+X);
1493 p->enc = ENC(db);
1494 p->db = db;
1495 p->nRef = 1;
1496 memset(&p[1], 0, nExtra);
1497 }else{
1498 return (KeyInfo*)sqlite3OomFault(db);
1499 }
1500 return p;
1501}
1502
1503/*
1504** Deallocate a KeyInfo object
1505*/
1506void sqlite3KeyInfoUnref(KeyInfo *p){
1507 if( p ){
1508 assert( p->db!=0 );
1509 assert( p->nRef>0 );
1510 p->nRef--;
1511 if( p->nRef==0 ) sqlite3DbNNFreeNN(p->db, p);
1512 }
1513}
1514
1515/*
1516** Make a new pointer to a KeyInfo object
1517*/
1518KeyInfo *sqlite3KeyInfoRef(KeyInfo *p){
1519 if( p ){
1520 assert( p->nRef>0 );
1521 p->nRef++;
1522 }
1523 return p;
1524}
1525
1526#ifdef SQLITE_DEBUG
1527/*
1528** Return TRUE if a KeyInfo object can be change. The KeyInfo object
1529** can only be changed if this is just a single reference to the object.
1530**
1531** This routine is used only inside of assert() statements.
1532*/
1533int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; }
1534#endif /* SQLITE_DEBUG */
1535
1536/*
1537** Given an expression list, generate a KeyInfo structure that records
1538** the collating sequence for each expression in that expression list.
1539**
1540** If the ExprList is an ORDER BY or GROUP BY clause then the resulting
1541** KeyInfo structure is appropriate for initializing a virtual index to
1542** implement that clause. If the ExprList is the result set of a SELECT
1543** then the KeyInfo structure is appropriate for initializing a virtual
1544** index to implement a DISTINCT test.
1545**
1546** Space to hold the KeyInfo structure is obtained from malloc. The calling
1547** function is responsible for seeing that this structure is eventually
1548** freed.
1549*/
1550KeyInfo *sqlite3KeyInfoFromExprList(
1551 Parse *pParse, /* Parsing context */
1552 ExprList *pList, /* Form the KeyInfo object from this ExprList */
1553 int iStart, /* Begin with this column of pList */
1554 int nExtra /* Add this many extra columns to the end */
1555){
1556 int nExpr;
1557 KeyInfo *pInfo;
1558 struct ExprList_item *pItem;
1559 sqlite3 *db = pParse->db;
1560 int i;
1561
1562 nExpr = pList->nExpr;
1563 pInfo = sqlite3KeyInfoAlloc(db, nExpr-iStart, nExtra+1);
1564 if( pInfo ){
1565 assert( sqlite3KeyInfoIsWriteable(pInfo) );
1566 for(i=iStart, pItem=pList->a+iStart; i<nExpr; i++, pItem++){
1567 pInfo->aColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr);
1568 pInfo->aSortFlags[i-iStart] = pItem->fg.sortFlags;
1569 }
1570 }
1571 return pInfo;
1572}
1573
1574/*
1575** Name of the connection operator, used for error messages.
1576*/
1577const char *sqlite3SelectOpName(int id){
1578 char *z;
1579 switch( id ){
1580 case TK_ALL: z = "UNION ALL"; break;
1581 case TK_INTERSECT: z = "INTERSECT"; break;
1582 case TK_EXCEPT: z = "EXCEPT"; break;
1583 default: z = "UNION"; break;
1584 }
1585 return z;
1586}
1587
1588#ifndef SQLITE_OMIT_EXPLAIN
1589/*
1590** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function
1591** is a no-op. Otherwise, it adds a single row of output to the EQP result,
1592** where the caption is of the form:
1593**
1594** "USE TEMP B-TREE FOR xxx"
1595**
1596** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which
1597** is determined by the zUsage argument.
1598*/
1599static void explainTempTable(Parse *pParse, const char *zUsage){
1600 ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s", zUsage));
1601}
1602
1603/*
1604** Assign expression b to lvalue a. A second, no-op, version of this macro
1605** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code
1606** in sqlite3Select() to assign values to structure member variables that
1607** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the
1608** code with #ifndef directives.
1609*/
1610# define explainSetInteger(a, b) a = b
1611
1612#else
1613/* No-op versions of the explainXXX() functions and macros. */
1614# define explainTempTable(y,z)
1615# define explainSetInteger(y,z)
1616#endif
1617
1618
1619/*
1620** If the inner loop was generated using a non-null pOrderBy argument,
1621** then the results were placed in a sorter. After the loop is terminated
1622** we need to run the sorter and output the results. The following
1623** routine generates the code needed to do that.
1624*/
1625static void generateSortTail(
1626 Parse *pParse, /* Parsing context */
1627 Select *p, /* The SELECT statement */
1628 SortCtx *pSort, /* Information on the ORDER BY clause */
1629 int nColumn, /* Number of columns of data */
1630 SelectDest *pDest /* Write the sorted results here */
1631){
1632 Vdbe *v = pParse->pVdbe; /* The prepared statement */
1633 int addrBreak = pSort->labelDone; /* Jump here to exit loop */
1634 int addrContinue = sqlite3VdbeMakeLabel(pParse);/* Jump here for next cycle */
1635 int addr; /* Top of output loop. Jump for Next. */
1636 int addrOnce = 0;
1637 int iTab;
1638 ExprList *pOrderBy = pSort->pOrderBy;
1639 int eDest = pDest->eDest;
1640 int iParm = pDest->iSDParm;
1641 int regRow;
1642 int regRowid;
1643 int iCol;
1644 int nKey; /* Number of key columns in sorter record */
1645 int iSortTab; /* Sorter cursor to read from */
1646 int i;
1647 int bSeq; /* True if sorter record includes seq. no. */
1648 int nRefKey = 0;
1649 struct ExprList_item *aOutEx = p->pEList->a;
1650
1651 assert( addrBreak<0 );
1652 if( pSort->labelBkOut ){
1653 sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
1654 sqlite3VdbeGoto(v, addrBreak);
1655 sqlite3VdbeResolveLabel(v, pSort->labelBkOut);
1656 }
1657
1658#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1659 /* Open any cursors needed for sorter-reference expressions */
1660 for(i=0; i<pSort->nDefer; i++){
1661 Table *pTab = pSort->aDefer[i].pTab;
1662 int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
1663 sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead);
1664 nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey);
1665 }
1666#endif
1667
1668 iTab = pSort->iECursor;
1669 if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){
1670 if( eDest==SRT_Mem && p->iOffset ){
1671 sqlite3VdbeAddOp2(v, OP_Null, 0, pDest->iSdst);
1672 }
1673 regRowid = 0;
1674 regRow = pDest->iSdst;
1675 }else{
1676 regRowid = sqlite3GetTempReg(pParse);
1677 if( eDest==SRT_EphemTab || eDest==SRT_Table ){
1678 regRow = sqlite3GetTempReg(pParse);
1679 nColumn = 0;
1680 }else{
1681 regRow = sqlite3GetTempRange(pParse, nColumn);
1682 }
1683 }
1684 nKey = pOrderBy->nExpr - pSort->nOBSat;
1685 if( pSort->sortFlags & SORTFLAG_UseSorter ){
1686 int regSortOut = ++pParse->nMem;
1687 iSortTab = pParse->nTab++;
1688 if( pSort->labelBkOut ){
1689 addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
1690 }
1691 sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut,
1692 nKey+1+nColumn+nRefKey);
1693 if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
1694 addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
1695 VdbeCoverage(v);
1696 assert( p->iLimit==0 && p->iOffset==0 );
1697 sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab);
1698 bSeq = 0;
1699 }else{
1700 addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
1701 codeOffset(v, p->iOffset, addrContinue);
1702 iSortTab = iTab;
1703 bSeq = 1;
1704 if( p->iOffset>0 ){
1705 sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
1706 }
1707 }
1708 for(i=0, iCol=nKey+bSeq-1; i<nColumn; i++){
1709#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1710 if( aOutEx[i].fg.bSorterRef ) continue;
1711#endif
1712 if( aOutEx[i].u.x.iOrderByCol==0 ) iCol++;
1713 }
1714#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1715 if( pSort->nDefer ){
1716 int iKey = iCol+1;
1717 int regKey = sqlite3GetTempRange(pParse, nRefKey);
1718
1719 for(i=0; i<pSort->nDefer; i++){
1720 int iCsr = pSort->aDefer[i].iCsr;
1721 Table *pTab = pSort->aDefer[i].pTab;
1722 int nKey = pSort->aDefer[i].nKey;
1723
1724 sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
1725 if( HasRowid(pTab) ){
1726 sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey);
1727 sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr,
1728 sqlite3VdbeCurrentAddr(v)+1, regKey);
1729 }else{
1730 int k;
1731 int iJmp;
1732 assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey );
1733 for(k=0; k<nKey; k++){
1734 sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey+k);
1735 }
1736 iJmp = sqlite3VdbeCurrentAddr(v);
1737 sqlite3VdbeAddOp4Int(v, OP_SeekGE, iCsr, iJmp+2, regKey, nKey);
1738 sqlite3VdbeAddOp4Int(v, OP_IdxLE, iCsr, iJmp+3, regKey, nKey);
1739 sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
1740 }
1741 }
1742 sqlite3ReleaseTempRange(pParse, regKey, nRefKey);
1743 }
1744#endif
1745 for(i=nColumn-1; i>=0; i--){
1746#ifdef SQLITE_ENABLE_SORTER_REFERENCES
1747 if( aOutEx[i].fg.bSorterRef ){
1748 sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i);
1749 }else
1750#endif
1751 {
1752 int iRead;
1753 if( aOutEx[i].u.x.iOrderByCol ){
1754 iRead = aOutEx[i].u.x.iOrderByCol-1;
1755 }else{
1756 iRead = iCol--;
1757 }
1758 sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
1759 VdbeComment((v, "%s", aOutEx[i].zEName));
1760 }
1761 }
1762 switch( eDest ){
1763 case SRT_Table:
1764 case SRT_EphemTab: {
1765 sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq, regRow);
1766 sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid);
1767 sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid);
1768 sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
1769 break;
1770 }
1771#ifndef SQLITE_OMIT_SUBQUERY
1772 case SRT_Set: {
1773 assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) );
1774 sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid,
1775 pDest->zAffSdst, nColumn);
1776 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, regRowid, regRow, nColumn);
1777 break;
1778 }
1779 case SRT_Mem: {
1780 /* The LIMIT clause will terminate the loop for us */
1781 break;
1782 }
1783#endif
1784 case SRT_Upfrom: {
1785 int i2 = pDest->iSDParm2;
1786 int r1 = sqlite3GetTempReg(pParse);
1787 sqlite3VdbeAddOp3(v, OP_MakeRecord,regRow+(i2<0),nColumn-(i2<0),r1);
1788 if( i2<0 ){
1789 sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regRow);
1790 }else{
1791 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regRow, i2);
1792 }
1793 break;
1794 }
1795 default: {
1796 assert( eDest==SRT_Output || eDest==SRT_Coroutine );
1797 testcase( eDest==SRT_Output );
1798 testcase( eDest==SRT_Coroutine );
1799 if( eDest==SRT_Output ){
1800 sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn);
1801 }else{
1802 sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm);
1803 }
1804 break;
1805 }
1806 }
1807 if( regRowid ){
1808 if( eDest==SRT_Set ){
1809 sqlite3ReleaseTempRange(pParse, regRow, nColumn);
1810 }else{
1811 sqlite3ReleaseTempReg(pParse, regRow);
1812 }
1813 sqlite3ReleaseTempReg(pParse, regRowid);
1814 }
1815 /* The bottom of the loop
1816 */
1817 sqlite3VdbeResolveLabel(v, addrContinue);
1818 if( pSort->sortFlags & SORTFLAG_UseSorter ){
1819 sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v);
1820 }else{
1821 sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v);
1822 }
1823 if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn);
1824 sqlite3VdbeResolveLabel(v, addrBreak);
1825}
1826
1827/*
1828** Return a pointer to a string containing the 'declaration type' of the
1829** expression pExpr. The string may be treated as static by the caller.
1830**
1831** The declaration type is the exact datatype definition extracted from the
1832** original CREATE TABLE statement if the expression is a column. The
1833** declaration type for a ROWID field is INTEGER. Exactly when an expression
1834** is considered a column can be complex in the presence of subqueries. The
1835** result-set expression in all of the following SELECT statements is
1836** considered a column by this function.
1837**
1838** SELECT col FROM tbl;
1839** SELECT (SELECT col FROM tbl;
1840** SELECT (SELECT col FROM tbl);
1841** SELECT abc FROM (SELECT col AS abc FROM tbl);
1842**
1843** The declaration type for any expression other than a column is NULL.
1844**
1845** This routine has either 3 or 6 parameters depending on whether or not
1846** the SQLITE_ENABLE_COLUMN_METADATA compile-time option is used.
1847*/
1848#ifdef SQLITE_ENABLE_COLUMN_METADATA
1849# define columnType(A,B,C,D,E) columnTypeImpl(A,B,C,D,E)
1850#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */
1851# define columnType(A,B,C,D,E) columnTypeImpl(A,B)
1852#endif
1853static const char *columnTypeImpl(
1854 NameContext *pNC,
1855#ifndef SQLITE_ENABLE_COLUMN_METADATA
1856 Expr *pExpr
1857#else
1858 Expr *pExpr,
1859 const char **pzOrigDb,
1860 const char **pzOrigTab,
1861 const char **pzOrigCol
1862#endif
1863){
1864 char const *zType = 0;
1865 int j;
1866#ifdef SQLITE_ENABLE_COLUMN_METADATA
1867 char const *zOrigDb = 0;
1868 char const *zOrigTab = 0;
1869 char const *zOrigCol = 0;
1870#endif
1871
1872 assert( pExpr!=0 );
1873 assert( pNC->pSrcList!=0 );
1874 switch( pExpr->op ){
1875 case TK_COLUMN: {
1876 /* The expression is a column. Locate the table the column is being
1877 ** extracted from in NameContext.pSrcList. This table may be real
1878 ** database table or a subquery.
1879 */
1880 Table *pTab = 0; /* Table structure column is extracted from */
1881 Select *pS = 0; /* Select the column is extracted from */
1882 int iCol = pExpr->iColumn; /* Index of column in pTab */
1883 while( pNC && !pTab ){
1884 SrcList *pTabList = pNC->pSrcList;
1885 for(j=0;j<pTabList->nSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++);
1886 if( j<pTabList->nSrc ){
1887 pTab = pTabList->a[j].pTab;
1888 pS = pTabList->a[j].pSelect;
1889 }else{
1890 pNC = pNC->pNext;
1891 }
1892 }
1893
1894 if( pTab==0 ){
1895 /* At one time, code such as "SELECT new.x" within a trigger would
1896 ** cause this condition to run. Since then, we have restructured how
1897 ** trigger code is generated and so this condition is no longer
1898 ** possible. However, it can still be true for statements like
1899 ** the following:
1900 **
1901 ** CREATE TABLE t1(col INTEGER);
1902 ** SELECT (SELECT t1.col) FROM FROM t1;
1903 **
1904 ** when columnType() is called on the expression "t1.col" in the
1905 ** sub-select. In this case, set the column type to NULL, even
1906 ** though it should really be "INTEGER".
1907 **
1908 ** This is not a problem, as the column type of "t1.col" is never
1909 ** used. When columnType() is called on the expression
1910 ** "(SELECT t1.col)", the correct type is returned (see the TK_SELECT
1911 ** branch below. */
1912 break;
1913 }
1914
1915 assert( pTab && ExprUseYTab(pExpr) && pExpr->y.pTab==pTab );
1916 if( pS ){
1917 /* The "table" is actually a sub-select or a view in the FROM clause
1918 ** of the SELECT statement. Return the declaration type and origin
1919 ** data for the result-set column of the sub-select.
1920 */
1921 if( iCol<pS->pEList->nExpr
1922#ifdef SQLITE_ALLOW_ROWID_IN_VIEW
1923 && iCol>=0
1924#else
1925 && ALWAYS(iCol>=0)
1926#endif
1927 ){
1928 /* If iCol is less than zero, then the expression requests the
1929 ** rowid of the sub-select or view. This expression is legal (see
1930 ** test case misc2.2.2) - it always evaluates to NULL.
1931 */
1932 NameContext sNC;
1933 Expr *p = pS->pEList->a[iCol].pExpr;
1934 sNC.pSrcList = pS->pSrc;
1935 sNC.pNext = pNC;
1936 sNC.pParse = pNC->pParse;
1937 zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol);
1938 }
1939 }else{
1940 /* A real table or a CTE table */
1941 assert( !pS );
1942#ifdef SQLITE_ENABLE_COLUMN_METADATA
1943 if( iCol<0 ) iCol = pTab->iPKey;
1944 assert( iCol==XN_ROWID || (iCol>=0 && iCol<pTab->nCol) );
1945 if( iCol<0 ){
1946 zType = "INTEGER";
1947 zOrigCol = "rowid";
1948 }else{
1949 zOrigCol = pTab->aCol[iCol].zCnName;
1950 zType = sqlite3ColumnType(&pTab->aCol[iCol],0);
1951 }
1952 zOrigTab = pTab->zName;
1953 if( pNC->pParse && pTab->pSchema ){
1954 int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema);
1955 zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName;
1956 }
1957#else
1958 assert( iCol==XN_ROWID || (iCol>=0 && iCol<pTab->nCol) );
1959 if( iCol<0 ){
1960 zType = "INTEGER";
1961 }else{
1962 zType = sqlite3ColumnType(&pTab->aCol[iCol],0);
1963 }
1964#endif
1965 }
1966 break;
1967 }
1968#ifndef SQLITE_OMIT_SUBQUERY
1969 case TK_SELECT: {
1970 /* The expression is a sub-select. Return the declaration type and
1971 ** origin info for the single column in the result set of the SELECT
1972 ** statement.
1973 */
1974 NameContext sNC;
1975 Select *pS;
1976 Expr *p;
1977 assert( ExprUseXSelect(pExpr) );
1978 pS = pExpr->x.pSelect;
1979 p = pS->pEList->a[0].pExpr;
1980 sNC.pSrcList = pS->pSrc;
1981 sNC.pNext = pNC;
1982 sNC.pParse = pNC->pParse;
1983 zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol);
1984 break;
1985 }
1986#endif
1987 }
1988
1989#ifdef SQLITE_ENABLE_COLUMN_METADATA
1990 if( pzOrigDb ){
1991 assert( pzOrigTab && pzOrigCol );
1992 *pzOrigDb = zOrigDb;
1993 *pzOrigTab = zOrigTab;
1994 *pzOrigCol = zOrigCol;
1995 }
1996#endif
1997 return zType;
1998}
1999
2000/*
2001** Generate code that will tell the VDBE the declaration types of columns
2002** in the result set.
2003*/
2004static void generateColumnTypes(
2005 Parse *pParse, /* Parser context */
2006 SrcList *pTabList, /* List of tables */
2007 ExprList *pEList /* Expressions defining the result set */
2008){
2009#ifndef SQLITE_OMIT_DECLTYPE
2010 Vdbe *v = pParse->pVdbe;
2011 int i;
2012 NameContext sNC;
2013 sNC.pSrcList = pTabList;
2014 sNC.pParse = pParse;
2015 sNC.pNext = 0;
2016 for(i=0; i<pEList->nExpr; i++){
2017 Expr *p = pEList->a[i].pExpr;
2018 const char *zType;
2019#ifdef SQLITE_ENABLE_COLUMN_METADATA
2020 const char *zOrigDb = 0;
2021 const char *zOrigTab = 0;
2022 const char *zOrigCol = 0;
2023 zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol);
2024
2025 /* The vdbe must make its own copy of the column-type and other
2026 ** column specific strings, in case the schema is reset before this
2027 ** virtual machine is deleted.
2028 */
2029 sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, SQLITE_TRANSIENT);
2030 sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT);
2031 sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT);
2032#else
2033 zType = columnType(&sNC, p, 0, 0, 0);
2034#endif
2035 sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT);
2036 }
2037#endif /* !defined(SQLITE_OMIT_DECLTYPE) */
2038}
2039
2040
2041/*
2042** Compute the column names for a SELECT statement.
2043**
2044** The only guarantee that SQLite makes about column names is that if the
2045** column has an AS clause assigning it a name, that will be the name used.
2046** That is the only documented guarantee. However, countless applications
2047** developed over the years have made baseless assumptions about column names
2048** and will break if those assumptions changes. Hence, use extreme caution
2049** when modifying this routine to avoid breaking legacy.
2050**
2051** See Also: sqlite3ColumnsFromExprList()
2052**
2053** The PRAGMA short_column_names and PRAGMA full_column_names settings are
2054** deprecated. The default setting is short=ON, full=OFF. 99.9% of all
2055** applications should operate this way. Nevertheless, we need to support the
2056** other modes for legacy:
2057**
2058** short=OFF, full=OFF: Column name is the text of the expression has it
2059** originally appears in the SELECT statement. In
2060** other words, the zSpan of the result expression.
2061**
2062** short=ON, full=OFF: (This is the default setting). If the result
2063** refers directly to a table column, then the
2064** result column name is just the table column
2065** name: COLUMN. Otherwise use zSpan.
2066**
2067** full=ON, short=ANY: If the result refers directly to a table column,
2068** then the result column name with the table name
2069** prefix, ex: TABLE.COLUMN. Otherwise use zSpan.
2070*/
2071void sqlite3GenerateColumnNames(
2072 Parse *pParse, /* Parser context */
2073 Select *pSelect /* Generate column names for this SELECT statement */
2074){
2075 Vdbe *v = pParse->pVdbe;
2076 int i;
2077 Table *pTab;
2078 SrcList *pTabList;
2079 ExprList *pEList;
2080 sqlite3 *db = pParse->db;
2081 int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */
2082 int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */
2083
2084#ifndef SQLITE_OMIT_EXPLAIN
2085 /* If this is an EXPLAIN, skip this step */
2086 if( pParse->explain ){
2087 return;
2088 }
2089#endif
2090
2091 if( pParse->colNamesSet ) return;
2092 /* Column names are determined by the left-most term of a compound select */
2093 while( pSelect->pPrior ) pSelect = pSelect->pPrior;
2094 SELECTTRACE(1,pParse,pSelect,("generating column names\n"));
2095 pTabList = pSelect->pSrc;
2096 pEList = pSelect->pEList;
2097 assert( v!=0 );
2098 assert( pTabList!=0 );
2099 pParse->colNamesSet = 1;
2100 fullName = (db->flags & SQLITE_FullColNames)!=0;
2101 srcName = (db->flags & SQLITE_ShortColNames)!=0 || fullName;
2102 sqlite3VdbeSetNumCols(v, pEList->nExpr);
2103 for(i=0; i<pEList->nExpr; i++){
2104 Expr *p = pEList->a[i].pExpr;
2105
2106 assert( p!=0 );
2107 assert( p->op!=TK_AGG_COLUMN ); /* Agg processing has not run yet */
2108 assert( p->op!=TK_COLUMN
2109 || (ExprUseYTab(p) && p->y.pTab!=0) ); /* Covering idx not yet coded */
2110 if( pEList->a[i].zEName && pEList->a[i].fg.eEName==ENAME_NAME ){
2111 /* An AS clause always takes first priority */
2112 char *zName = pEList->a[i].zEName;
2113 sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT);
2114 }else if( srcName && p->op==TK_COLUMN ){
2115 char *zCol;
2116 int iCol = p->iColumn;
2117 pTab = p->y.pTab;
2118 assert( pTab!=0 );
2119 if( iCol<0 ) iCol = pTab->iPKey;
2120 assert( iCol==-1 || (iCol>=0 && iCol<pTab->nCol) );
2121 if( iCol<0 ){
2122 zCol = "rowid";
2123 }else{
2124 zCol = pTab->aCol[iCol].zCnName;
2125 }
2126 if( fullName ){
2127 char *zName = 0;
2128 zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol);
2129 sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC);
2130 }else{
2131 sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT);
2132 }
2133 }else{
2134 const char *z = pEList->a[i].zEName;
2135 z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z);
2136 sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC);
2137 }
2138 }
2139 generateColumnTypes(pParse, pTabList, pEList);
2140}
2141
2142/*
2143** Given an expression list (which is really the list of expressions
2144** that form the result set of a SELECT statement) compute appropriate
2145** column names for a table that would hold the expression list.
2146**
2147** All column names will be unique.
2148**
2149** Only the column names are computed. Column.zType, Column.zColl,
2150** and other fields of Column are zeroed.
2151**
2152** Return SQLITE_OK on success. If a memory allocation error occurs,
2153** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM.
2154**
2155** The only guarantee that SQLite makes about column names is that if the
2156** column has an AS clause assigning it a name, that will be the name used.
2157** That is the only documented guarantee. However, countless applications
2158** developed over the years have made baseless assumptions about column names
2159** and will break if those assumptions changes. Hence, use extreme caution
2160** when modifying this routine to avoid breaking legacy.
2161**
2162** See Also: sqlite3GenerateColumnNames()
2163*/
2164int sqlite3ColumnsFromExprList(
2165 Parse *pParse, /* Parsing context */
2166 ExprList *pEList, /* Expr list from which to derive column names */
2167 i16 *pnCol, /* Write the number of columns here */
2168 Column **paCol /* Write the new column list here */
2169){
2170 sqlite3 *db = pParse->db; /* Database connection */
2171 int i, j; /* Loop counters */
2172 u32 cnt; /* Index added to make the name unique */
2173 Column *aCol, *pCol; /* For looping over result columns */
2174 int nCol; /* Number of columns in the result set */
2175 char *zName; /* Column name */
2176 int nName; /* Size of name in zName[] */
2177 Hash ht; /* Hash table of column names */
2178 Table *pTab;
2179
2180 sqlite3HashInit(&ht);
2181 if( pEList ){
2182 nCol = pEList->nExpr;
2183 aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol);
2184 testcase( aCol==0 );
2185 if( NEVER(nCol>32767) ) nCol = 32767;
2186 }else{
2187 nCol = 0;
2188 aCol = 0;
2189 }
2190 assert( nCol==(i16)nCol );
2191 *pnCol = nCol;
2192 *paCol = aCol;
2193
2194 for(i=0, pCol=aCol; i<nCol && !db->mallocFailed; i++, pCol++){
2195 struct ExprList_item *pX = &pEList->a[i];
2196 struct ExprList_item *pCollide;
2197 /* Get an appropriate name for the column
2198 */
2199 if( (zName = pX->zEName)!=0 && pX->fg.eEName==ENAME_NAME ){
2200 /* If the column contains an "AS <name>" phrase, use <name> as the name */
2201 }else{
2202 Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pX->pExpr);
2203 while( ALWAYS(pColExpr!=0) && pColExpr->op==TK_DOT ){
2204 pColExpr = pColExpr->pRight;
2205 assert( pColExpr!=0 );
2206 }
2207 if( pColExpr->op==TK_COLUMN
2208 && ALWAYS( ExprUseYTab(pColExpr) )
2209 && ALWAYS( pColExpr->y.pTab!=0 )
2210 ){
2211 /* For columns use the column name name */
2212 int iCol = pColExpr->iColumn;
2213 pTab = pColExpr->y.pTab;
2214 if( iCol<0 ) iCol = pTab->iPKey;
2215 zName = iCol>=0 ? pTab->aCol[iCol].zCnName : "rowid";
2216 }else if( pColExpr->op==TK_ID ){
2217 assert( !ExprHasProperty(pColExpr, EP_IntValue) );
2218 zName = pColExpr->u.zToken;
2219 }else{
2220 /* Use the original text of the column expression as its name */
2221 assert( zName==pX->zEName ); /* pointer comparison intended */
2222 }
2223 }
2224 if( zName && !sqlite3IsTrueOrFalse(zName) ){
2225 zName = sqlite3DbStrDup(db, zName);
2226 }else{
2227 zName = sqlite3MPrintf(db,"column%d",i+1);
2228 }
2229
2230 /* Make sure the column name is unique. If the name is not unique,
2231 ** append an integer to the name so that it becomes unique.
2232 */
2233 cnt = 0;
2234 while( zName && (pCollide = sqlite3HashFind(&ht, zName))!=0 ){
2235 if( pCollide->fg.bUsingTerm ){
2236 pCol->colFlags |= COLFLAG_NOEXPAND;
2237 }
2238 nName = sqlite3Strlen30(zName);
2239 if( nName>0 ){
2240 for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){}
2241 if( zName[j]==':' ) nName = j;
2242 }
2243 zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt);
2244 if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt);
2245 }
2246 pCol->zCnName = zName;
2247 pCol->hName = sqlite3StrIHash(zName);
2248 if( pX->fg.bNoExpand ){
2249 pCol->colFlags |= COLFLAG_NOEXPAND;
2250 }
2251 sqlite3ColumnPropertiesFromName(0, pCol);
2252 if( zName && sqlite3HashInsert(&ht, zName, pX)==pX ){
2253 sqlite3OomFault(db);
2254 }
2255 }
2256 sqlite3HashClear(&ht);
2257 if( db->mallocFailed ){
2258 for(j=0; j<i; j++){
2259 sqlite3DbFree(db, aCol[j].zCnName);
2260 }
2261 sqlite3DbFree(db, aCol);
2262 *paCol = 0;
2263 *pnCol = 0;
2264 return SQLITE_NOMEM_BKPT;
2265 }
2266 return SQLITE_OK;
2267}
2268
2269/*
2270** Add type and collation information to a column list based on
2271** a SELECT statement.
2272**
2273** The column list presumably came from selectColumnNamesFromExprList().
2274** The column list has only names, not types or collations. This
2275** routine goes through and adds the types and collations.
2276**
2277** This routine requires that all identifiers in the SELECT
2278** statement be resolved.
2279*/
2280void sqlite3SelectAddColumnTypeAndCollation(
2281 Parse *pParse, /* Parsing contexts */
2282 Table *pTab, /* Add column type information to this table */
2283 Select *pSelect, /* SELECT used to determine types and collations */
2284 char aff /* Default affinity for columns */
2285){
2286 sqlite3 *db = pParse->db;
2287 NameContext sNC;
2288 Column *pCol;
2289 CollSeq *pColl;
2290 int i;
2291 Expr *p;
2292 struct ExprList_item *a;
2293
2294 assert( pSelect!=0 );
2295 assert( (pSelect->selFlags & SF_Resolved)!=0 );
2296 assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed );
2297 if( db->mallocFailed ) return;
2298 memset(&sNC, 0, sizeof(sNC));
2299 sNC.pSrcList = pSelect->pSrc;
2300 a = pSelect->pEList->a;
2301 for(i=0, pCol=pTab->aCol; i<pTab->nCol; i++, pCol++){
2302 const char *zType;
2303 i64 n, m;
2304 pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT);
2305 p = a[i].pExpr;
2306 zType = columnType(&sNC, p, 0, 0, 0);
2307 /* pCol->szEst = ... // Column size est for SELECT tables never used */
2308 pCol->affinity = sqlite3ExprAffinity(p);
2309 if( zType ){
2310 m = sqlite3Strlen30(zType);
2311 n = sqlite3Strlen30(pCol->zCnName);
2312 pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2);
2313 if( pCol->zCnName ){
2314 memcpy(&pCol->zCnName[n+1], zType, m+1);
2315 pCol->colFlags |= COLFLAG_HASTYPE;
2316 }else{
2317 testcase( pCol->colFlags & COLFLAG_HASTYPE );
2318 pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL);
2319 }
2320 }
2321 if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff;
2322 pColl = sqlite3ExprCollSeq(pParse, p);
2323 if( pColl ){
2324 assert( pTab->pIndex==0 );
2325 sqlite3ColumnSetColl(db, pCol, pColl->zName);
2326 }
2327 }
2328 pTab->szTabRow = 1; /* Any non-zero value works */
2329}
2330
2331/*
2332** Given a SELECT statement, generate a Table structure that describes
2333** the result set of that SELECT.
2334*/
2335Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, char aff){
2336 Table *pTab;
2337 sqlite3 *db = pParse->db;
2338 u64 savedFlags;
2339
2340 savedFlags = db->flags;
2341 db->flags &= ~(u64)SQLITE_FullColNames;
2342 db->flags |= SQLITE_ShortColNames;
2343 sqlite3SelectPrep(pParse, pSelect, 0);
2344 db->flags = savedFlags;
2345 if( pParse->nErr ) return 0;
2346 while( pSelect->pPrior ) pSelect = pSelect->pPrior;
2347 pTab = sqlite3DbMallocZero(db, sizeof(Table) );
2348 if( pTab==0 ){
2349 return 0;
2350 }
2351 pTab->nTabRef = 1;
2352 pTab->zName = 0;
2353 pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );
2354 sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol);
2355 sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff);
2356 pTab->iPKey = -1;
2357 if( db->mallocFailed ){
2358 sqlite3DeleteTable(db, pTab);
2359 return 0;
2360 }
2361 return pTab;
2362}
2363
2364/*
2365** Get a VDBE for the given parser context. Create a new one if necessary.
2366** If an error occurs, return NULL and leave a message in pParse.
2367*/
2368Vdbe *sqlite3GetVdbe(Parse *pParse){
2369 if( pParse->pVdbe ){
2370 return pParse->pVdbe;
2371 }
2372 if( pParse->pToplevel==0
2373 && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst)
2374 ){
2375 pParse->okConstFactor = 1;
2376 }
2377 return sqlite3VdbeCreate(pParse);
2378}
2379
2380
2381/*
2382** Compute the iLimit and iOffset fields of the SELECT based on the
2383** pLimit expressions. pLimit->pLeft and pLimit->pRight hold the expressions
2384** that appear in the original SQL statement after the LIMIT and OFFSET
2385** keywords. Or NULL if those keywords are omitted. iLimit and iOffset
2386** are the integer memory register numbers for counters used to compute
2387** the limit and offset. If there is no limit and/or offset, then
2388** iLimit and iOffset are negative.
2389**
2390** This routine changes the values of iLimit and iOffset only if
2391** a limit or offset is defined by pLimit->pLeft and pLimit->pRight. iLimit
2392** and iOffset should have been preset to appropriate default values (zero)
2393** prior to calling this routine.
2394**
2395** The iOffset register (if it exists) is initialized to the value
2396** of the OFFSET. The iLimit register is initialized to LIMIT. Register
2397** iOffset+1 is initialized to LIMIT+OFFSET.
2398**
2399** Only if pLimit->pLeft!=0 do the limit registers get
2400** redefined. The UNION ALL operator uses this property to force
2401** the reuse of the same limit and offset registers across multiple
2402** SELECT statements.
2403*/
2404static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
2405 Vdbe *v = 0;
2406 int iLimit = 0;
2407 int iOffset;
2408 int n;
2409 Expr *pLimit = p->pLimit;
2410
2411 if( p->iLimit ) return;
2412
2413 /*
2414 ** "LIMIT -1" always shows all rows. There is some
2415 ** controversy about what the correct behavior should be.
2416 ** The current implementation interprets "LIMIT 0" to mean
2417 ** no rows.
2418 */
2419 if( pLimit ){
2420 assert( pLimit->op==TK_LIMIT );
2421 assert( pLimit->pLeft!=0 );
2422 p->iLimit = iLimit = ++pParse->nMem;
2423 v = sqlite3GetVdbe(pParse);
2424 assert( v!=0 );
2425 if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){
2426 sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
2427 VdbeComment((v, "LIMIT counter"));
2428 if( n==0 ){
2429 sqlite3VdbeGoto(v, iBreak);
2430 }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){
2431 p->nSelectRow = sqlite3LogEst((u64)n);
2432 p->selFlags |= SF_FixedLimit;
2433 }
2434 }else{
2435 sqlite3ExprCode(pParse, pLimit->pLeft, iLimit);
2436 sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v);
2437 VdbeComment((v, "LIMIT counter"));
2438 sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v);
2439 }
2440 if( pLimit->pRight ){
2441 p->iOffset = iOffset = ++pParse->nMem;
2442 pParse->nMem++; /* Allocate an extra register for limit+offset */
2443 sqlite3ExprCode(pParse, pLimit->pRight, iOffset);
2444 sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v);
2445 VdbeComment((v, "OFFSET counter"));
2446 sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset);
2447 VdbeComment((v, "LIMIT+OFFSET"));
2448 }
2449 }
2450}
2451
2452#ifndef SQLITE_OMIT_COMPOUND_SELECT
2453/*
2454** Return the appropriate collating sequence for the iCol-th column of
2455** the result set for the compound-select statement "p". Return NULL if
2456** the column has no default collating sequence.
2457**
2458** The collating sequence for the compound select is taken from the
2459** left-most term of the select that has a collating sequence.
2460*/
2461static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){
2462 CollSeq *pRet;
2463 if( p->pPrior ){
2464 pRet = multiSelectCollSeq(pParse, p->pPrior, iCol);
2465 }else{
2466 pRet = 0;
2467 }
2468 assert( iCol>=0 );
2469 /* iCol must be less than p->pEList->nExpr. Otherwise an error would
2470 ** have been thrown during name resolution and we would not have gotten
2471 ** this far */
2472 if( pRet==0 && ALWAYS(iCol<p->pEList->nExpr) ){
2473 pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr);
2474 }
2475 return pRet;
2476}
2477
2478/*
2479** The select statement passed as the second parameter is a compound SELECT
2480** with an ORDER BY clause. This function allocates and returns a KeyInfo
2481** structure suitable for implementing the ORDER BY.
2482**
2483** Space to hold the KeyInfo structure is obtained from malloc. The calling
2484** function is responsible for ensuring that this structure is eventually
2485** freed.
2486*/
2487static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){
2488 ExprList *pOrderBy = p->pOrderBy;
2489 int nOrderBy = ALWAYS(pOrderBy!=0) ? pOrderBy->nExpr : 0;
2490 sqlite3 *db = pParse->db;
2491 KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1);
2492 if( pRet ){
2493 int i;
2494 for(i=0; i<nOrderBy; i++){
2495 struct ExprList_item *pItem = &pOrderBy->a[i];
2496 Expr *pTerm = pItem->pExpr;
2497 CollSeq *pColl;
2498
2499 if( pTerm->flags & EP_Collate ){
2500 pColl = sqlite3ExprCollSeq(pParse, pTerm);
2501 }else{
2502 pColl = multiSelectCollSeq(pParse, p, pItem->u.x.iOrderByCol-1);
2503 if( pColl==0 ) pColl = db->pDfltColl;
2504 pOrderBy->a[i].pExpr =
2505 sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName);
2506 }
2507 assert( sqlite3KeyInfoIsWriteable(pRet) );
2508 pRet->aColl[i] = pColl;
2509 pRet->aSortFlags[i] = pOrderBy->a[i].fg.sortFlags;
2510 }
2511 }
2512
2513 return pRet;
2514}
2515
2516#ifndef SQLITE_OMIT_CTE
2517/*
2518** This routine generates VDBE code to compute the content of a WITH RECURSIVE
2519** query of the form:
2520**
2521** <recursive-table> AS (<setup-query> UNION [ALL] <recursive-query>)
2522** \___________/ \_______________/
2523** p->pPrior p
2524**
2525**
2526** There is exactly one reference to the recursive-table in the FROM clause
2527** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag.
2528**
2529** The setup-query runs once to generate an initial set of rows that go
2530** into a Queue table. Rows are extracted from the Queue table one by
2531** one. Each row extracted from Queue is output to pDest. Then the single
2532** extracted row (now in the iCurrent table) becomes the content of the
2533** recursive-table for a recursive-query run. The output of the recursive-query
2534** is added back into the Queue table. Then another row is extracted from Queue
2535** and the iteration continues until the Queue table is empty.
2536**
2537** If the compound query operator is UNION then no duplicate rows are ever
2538** inserted into the Queue table. The iDistinct table keeps a copy of all rows
2539** that have ever been inserted into Queue and causes duplicates to be
2540** discarded. If the operator is UNION ALL, then duplicates are allowed.
2541**
2542** If the query has an ORDER BY, then entries in the Queue table are kept in
2543** ORDER BY order and the first entry is extracted for each cycle. Without
2544** an ORDER BY, the Queue table is just a FIFO.
2545**
2546** If a LIMIT clause is provided, then the iteration stops after LIMIT rows
2547** have been output to pDest. A LIMIT of zero means to output no rows and a
2548** negative LIMIT means to output all rows. If there is also an OFFSET clause
2549** with a positive value, then the first OFFSET outputs are discarded rather
2550** than being sent to pDest. The LIMIT count does not begin until after OFFSET
2551** rows have been skipped.
2552*/
2553static void generateWithRecursiveQuery(
2554 Parse *pParse, /* Parsing context */
2555 Select *p, /* The recursive SELECT to be coded */
2556 SelectDest *pDest /* What to do with query results */
2557){
2558 SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */
2559 int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */
2560 Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */
2561 Select *pSetup; /* The setup query */
2562 Select *pFirstRec; /* Left-most recursive term */
2563 int addrTop; /* Top of the loop */
2564 int addrCont, addrBreak; /* CONTINUE and BREAK addresses */
2565 int iCurrent = 0; /* The Current table */
2566 int regCurrent; /* Register holding Current table */
2567 int iQueue; /* The Queue table */
2568 int iDistinct = 0; /* To ensure unique results if UNION */
2569 int eDest = SRT_Fifo; /* How to write to Queue */
2570 SelectDest destQueue; /* SelectDest targetting the Queue table */
2571 int i; /* Loop counter */
2572 int rc; /* Result code */
2573 ExprList *pOrderBy; /* The ORDER BY clause */
2574 Expr *pLimit; /* Saved LIMIT and OFFSET */
2575 int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */
2576
2577#ifndef SQLITE_OMIT_WINDOWFUNC
2578 if( p->pWin ){
2579 sqlite3ErrorMsg(pParse, "cannot use window functions in recursive queries");
2580 return;
2581 }
2582#endif
2583
2584 /* Obtain authorization to do a recursive query */
2585 if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return;
2586
2587 /* Process the LIMIT and OFFSET clauses, if they exist */
2588 addrBreak = sqlite3VdbeMakeLabel(pParse);
2589 p->nSelectRow = 320; /* 4 billion rows */
2590 computeLimitRegisters(pParse, p, addrBreak);
2591 pLimit = p->pLimit;
2592 regLimit = p->iLimit;
2593 regOffset = p->iOffset;
2594 p->pLimit = 0;
2595 p->iLimit = p->iOffset = 0;
2596 pOrderBy = p->pOrderBy;
2597
2598 /* Locate the cursor number of the Current table */
2599 for(i=0; ALWAYS(i<pSrc->nSrc); i++){
2600 if( pSrc->a[i].fg.isRecursive ){
2601 iCurrent = pSrc->a[i].iCursor;
2602 break;
2603 }
2604 }
2605
2606 /* Allocate cursors numbers for Queue and Distinct. The cursor number for
2607 ** the Distinct table must be exactly one greater than Queue in order
2608 ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */
2609 iQueue = pParse->nTab++;
2610 if( p->op==TK_UNION ){
2611 eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo;
2612 iDistinct = pParse->nTab++;
2613 }else{
2614 eDest = pOrderBy ? SRT_Queue : SRT_Fifo;
2615 }
2616 sqlite3SelectDestInit(&destQueue, eDest, iQueue);
2617
2618 /* Allocate cursors for Current, Queue, and Distinct. */
2619 regCurrent = ++pParse->nMem;
2620 sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol);
2621 if( pOrderBy ){
2622 KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1);
2623 sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0,
2624 (char*)pKeyInfo, P4_KEYINFO);
2625 destQueue.pOrderBy = pOrderBy;
2626 }else{
2627 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol);
2628 }
2629 VdbeComment((v, "Queue table"));
2630 if( iDistinct ){
2631 p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0);
2632 p->selFlags |= SF_UsesEphemeral;
2633 }
2634
2635 /* Detach the ORDER BY clause from the compound SELECT */
2636 p->pOrderBy = 0;
2637
2638 /* Figure out how many elements of the compound SELECT are part of the
2639 ** recursive query. Make sure no recursive elements use aggregate
2640 ** functions. Mark the recursive elements as UNION ALL even if they
2641 ** are really UNION because the distinctness will be enforced by the
2642 ** iDistinct table. pFirstRec is left pointing to the left-most
2643 ** recursive term of the CTE.
2644 */
2645 for(pFirstRec=p; ALWAYS(pFirstRec!=0); pFirstRec=pFirstRec->pPrior){
2646 if( pFirstRec->selFlags & SF_Aggregate ){
2647 sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported");
2648 goto end_of_recursive_query;
2649 }
2650 pFirstRec->op = TK_ALL;
2651 if( (pFirstRec->pPrior->selFlags & SF_Recursive)==0 ) break;
2652 }
2653
2654 /* Store the results of the setup-query in Queue. */
2655 pSetup = pFirstRec->pPrior;
2656 pSetup->pNext = 0;
2657 ExplainQueryPlan((pParse, 1, "SETUP"));
2658 rc = sqlite3Select(pParse, pSetup, &destQueue);
2659 pSetup->pNext = p;
2660 if( rc ) goto end_of_recursive_query;
2661
2662 /* Find the next row in the Queue and output that row */
2663 addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); VdbeCoverage(v);
2664
2665 /* Transfer the next row in Queue over to Current */
2666 sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */
2667 if( pOrderBy ){
2668 sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent);
2669 }else{
2670 sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent);
2671 }
2672 sqlite3VdbeAddOp1(v, OP_Delete, iQueue);
2673
2674 /* Output the single row in Current */
2675 addrCont = sqlite3VdbeMakeLabel(pParse);
2676 codeOffset(v, regOffset, addrCont);
2677 selectInnerLoop(pParse, p, iCurrent,
2678 0, 0, pDest, addrCont, addrBreak);
2679 if( regLimit ){
2680 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, regLimit, addrBreak);
2681 VdbeCoverage(v);
2682 }
2683 sqlite3VdbeResolveLabel(v, addrCont);
2684
2685 /* Execute the recursive SELECT taking the single row in Current as
2686 ** the value for the recursive-table. Store the results in the Queue.
2687 */
2688 pFirstRec->pPrior = 0;
2689 ExplainQueryPlan((pParse, 1, "RECURSIVE STEP"));
2690 sqlite3Select(pParse, p, &destQueue);
2691 assert( pFirstRec->pPrior==0 );
2692 pFirstRec->pPrior = pSetup;
2693
2694 /* Keep running the loop until the Queue is empty */
2695 sqlite3VdbeGoto(v, addrTop);
2696 sqlite3VdbeResolveLabel(v, addrBreak);
2697
2698end_of_recursive_query:
2699 sqlite3ExprListDelete(pParse->db, p->pOrderBy);
2700 p->pOrderBy = pOrderBy;
2701 p->pLimit = pLimit;
2702 return;
2703}
2704#endif /* SQLITE_OMIT_CTE */
2705
2706/* Forward references */
2707static int multiSelectOrderBy(
2708 Parse *pParse, /* Parsing context */
2709 Select *p, /* The right-most of SELECTs to be coded */
2710 SelectDest *pDest /* What to do with query results */
2711);
2712
2713/*
2714** Handle the special case of a compound-select that originates from a
2715** VALUES clause. By handling this as a special case, we avoid deep
2716** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT
2717** on a VALUES clause.
2718**
2719** Because the Select object originates from a VALUES clause:
2720** (1) There is no LIMIT or OFFSET or else there is a LIMIT of exactly 1
2721** (2) All terms are UNION ALL
2722** (3) There is no ORDER BY clause
2723**
2724** The "LIMIT of exactly 1" case of condition (1) comes about when a VALUES
2725** clause occurs within scalar expression (ex: "SELECT (VALUES(1),(2),(3))").
2726** The sqlite3CodeSubselect will have added the LIMIT 1 clause in tht case.
2727** Since the limit is exactly 1, we only need to evaluate the left-most VALUES.
2728*/
2729static int multiSelectValues(
2730 Parse *pParse, /* Parsing context */
2731 Select *p, /* The right-most of SELECTs to be coded */
2732 SelectDest *pDest /* What to do with query results */
2733){
2734 int nRow = 1;
2735 int rc = 0;
2736 int bShowAll = p->pLimit==0;
2737 assert( p->selFlags & SF_MultiValue );
2738 do{
2739 assert( p->selFlags & SF_Values );
2740 assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) );
2741 assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr );
2742#ifndef SQLITE_OMIT_WINDOWFUNC
2743 if( p->pWin ) return -1;
2744#endif
2745 if( p->pPrior==0 ) break;
2746 assert( p->pPrior->pNext==p );
2747 p = p->pPrior;
2748 nRow += bShowAll;
2749 }while(1);
2750 ExplainQueryPlan((pParse, 0, "SCAN %d CONSTANT ROW%s", nRow,
2751 nRow==1 ? "" : "S"));
2752 while( p ){
2753 selectInnerLoop(pParse, p, -1, 0, 0, pDest, 1, 1);
2754 if( !bShowAll ) break;
2755 p->nSelectRow = nRow;
2756 p = p->pNext;
2757 }
2758 return rc;
2759}
2760
2761/*
2762** Return true if the SELECT statement which is known to be the recursive
2763** part of a recursive CTE still has its anchor terms attached. If the
2764** anchor terms have already been removed, then return false.
2765*/
2766static int hasAnchor(Select *p){
2767 while( p && (p->selFlags & SF_Recursive)!=0 ){ p = p->pPrior; }
2768 return p!=0;
2769}
2770
2771/*
2772** This routine is called to process a compound query form from
2773** two or more separate queries using UNION, UNION ALL, EXCEPT, or
2774** INTERSECT
2775**
2776** "p" points to the right-most of the two queries. the query on the
2777** left is p->pPrior. The left query could also be a compound query
2778** in which case this routine will be called recursively.
2779**
2780** The results of the total query are to be written into a destination
2781** of type eDest with parameter iParm.
2782**
2783** Example 1: Consider a three-way compound SQL statement.
2784**
2785** SELECT a FROM t1 UNION SELECT b FROM t2 UNION SELECT c FROM t3
2786**
2787** This statement is parsed up as follows:
2788**
2789** SELECT c FROM t3
2790** |
2791** `-----> SELECT b FROM t2
2792** |
2793** `------> SELECT a FROM t1
2794**
2795** The arrows in the diagram above represent the Select.pPrior pointer.
2796** So if this routine is called with p equal to the t3 query, then
2797** pPrior will be the t2 query. p->op will be TK_UNION in this case.
2798**
2799** Notice that because of the way SQLite parses compound SELECTs, the
2800** individual selects always group from left to right.
2801*/
2802static int multiSelect(
2803 Parse *pParse, /* Parsing context */
2804 Select *p, /* The right-most of SELECTs to be coded */
2805 SelectDest *pDest /* What to do with query results */
2806){
2807 int rc = SQLITE_OK; /* Success code from a subroutine */
2808 Select *pPrior; /* Another SELECT immediately to our left */
2809 Vdbe *v; /* Generate code to this VDBE */
2810 SelectDest dest; /* Alternative data destination */
2811 Select *pDelete = 0; /* Chain of simple selects to delete */
2812 sqlite3 *db; /* Database connection */
2813
2814 /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only
2815 ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT.
2816 */
2817 assert( p && p->pPrior ); /* Calling function guarantees this much */
2818 assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION );
2819 assert( p->selFlags & SF_Compound );
2820 db = pParse->db;
2821 pPrior = p->pPrior;
2822 dest = *pDest;
2823 assert( pPrior->pOrderBy==0 );
2824 assert( pPrior->pLimit==0 );
2825
2826 v = sqlite3GetVdbe(pParse);
2827 assert( v!=0 ); /* The VDBE already created by calling function */
2828
2829 /* Create the destination temporary table if necessary
2830 */
2831 if( dest.eDest==SRT_EphemTab ){
2832 assert( p->pEList );
2833 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr);
2834 dest.eDest = SRT_Table;
2835 }
2836
2837 /* Special handling for a compound-select that originates as a VALUES clause.
2838 */
2839 if( p->selFlags & SF_MultiValue ){
2840 rc = multiSelectValues(pParse, p, &dest);
2841 if( rc>=0 ) goto multi_select_end;
2842 rc = SQLITE_OK;
2843 }
2844
2845 /* Make sure all SELECTs in the statement have the same number of elements
2846 ** in their result sets.
2847 */
2848 assert( p->pEList && pPrior->pEList );
2849 assert( p->pEList->nExpr==pPrior->pEList->nExpr );
2850
2851#ifndef SQLITE_OMIT_CTE
2852 if( (p->selFlags & SF_Recursive)!=0 && hasAnchor(p) ){
2853 generateWithRecursiveQuery(pParse, p, &dest);
2854 }else
2855#endif
2856
2857 /* Compound SELECTs that have an ORDER BY clause are handled separately.
2858 */
2859 if( p->pOrderBy ){
2860 return multiSelectOrderBy(pParse, p, pDest);
2861 }else{
2862
2863#ifndef SQLITE_OMIT_EXPLAIN
2864 if( pPrior->pPrior==0 ){
2865 ExplainQueryPlan((pParse, 1, "COMPOUND QUERY"));
2866 ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY"));
2867 }
2868#endif
2869
2870 /* Generate code for the left and right SELECT statements.
2871 */
2872 switch( p->op ){
2873 case TK_ALL: {
2874 int addr = 0;
2875 int nLimit = 0; /* Initialize to suppress harmless compiler warning */
2876 assert( !pPrior->pLimit );
2877 pPrior->iLimit = p->iLimit;
2878 pPrior->iOffset = p->iOffset;
2879 pPrior->pLimit = p->pLimit;
2880 SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL left...\n"));
2881 rc = sqlite3Select(pParse, pPrior, &dest);
2882 pPrior->pLimit = 0;
2883 if( rc ){
2884 goto multi_select_end;
2885 }
2886 p->pPrior = 0;
2887 p->iLimit = pPrior->iLimit;
2888 p->iOffset = pPrior->iOffset;
2889 if( p->iLimit ){
2890 addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v);
2891 VdbeComment((v, "Jump ahead if LIMIT reached"));
2892 if( p->iOffset ){
2893 sqlite3VdbeAddOp3(v, OP_OffsetLimit,
2894 p->iLimit, p->iOffset+1, p->iOffset);
2895 }
2896 }
2897 ExplainQueryPlan((pParse, 1, "UNION ALL"));
2898 SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL right...\n"));
2899 rc = sqlite3Select(pParse, p, &dest);
2900 testcase( rc!=SQLITE_OK );
2901 pDelete = p->pPrior;
2902 p->pPrior = pPrior;
2903 p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
2904 if( p->pLimit
2905 && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit)
2906 && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit)
2907 ){
2908 p->nSelectRow = sqlite3LogEst((u64)nLimit);
2909 }
2910 if( addr ){
2911 sqlite3VdbeJumpHere(v, addr);
2912 }
2913 break;
2914 }
2915 case TK_EXCEPT:
2916 case TK_UNION: {
2917 int unionTab; /* Cursor number of the temp table holding result */
2918 u8 op = 0; /* One of the SRT_ operations to apply to self */
2919 int priorOp; /* The SRT_ operation to apply to prior selects */
2920 Expr *pLimit; /* Saved values of p->nLimit */
2921 int addr;
2922 SelectDest uniondest;
2923
2924 testcase( p->op==TK_EXCEPT );
2925 testcase( p->op==TK_UNION );
2926 priorOp = SRT_Union;
2927 if( dest.eDest==priorOp ){
2928 /* We can reuse a temporary table generated by a SELECT to our
2929 ** right.
2930 */
2931 assert( p->pLimit==0 ); /* Not allowed on leftward elements */
2932 unionTab = dest.iSDParm;
2933 }else{
2934 /* We will need to create our own temporary table to hold the
2935 ** intermediate results.
2936 */
2937 unionTab = pParse->nTab++;
2938 assert( p->pOrderBy==0 );
2939 addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0);
2940 assert( p->addrOpenEphm[0] == -1 );
2941 p->addrOpenEphm[0] = addr;
2942 findRightmost(p)->selFlags |= SF_UsesEphemeral;
2943 assert( p->pEList );
2944 }
2945
2946
2947 /* Code the SELECT statements to our left
2948 */
2949 assert( !pPrior->pOrderBy );
2950 sqlite3SelectDestInit(&uniondest, priorOp, unionTab);
2951 SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION left...\n"));
2952 rc = sqlite3Select(pParse, pPrior, &uniondest);
2953 if( rc ){
2954 goto multi_select_end;
2955 }
2956
2957 /* Code the current SELECT statement
2958 */
2959 if( p->op==TK_EXCEPT ){
2960 op = SRT_Except;
2961 }else{
2962 assert( p->op==TK_UNION );
2963 op = SRT_Union;
2964 }
2965 p->pPrior = 0;
2966 pLimit = p->pLimit;
2967 p->pLimit = 0;
2968 uniondest.eDest = op;
2969 ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE",
2970 sqlite3SelectOpName(p->op)));
2971 SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION right...\n"));
2972 rc = sqlite3Select(pParse, p, &uniondest);
2973 testcase( rc!=SQLITE_OK );
2974 assert( p->pOrderBy==0 );
2975 pDelete = p->pPrior;
2976 p->pPrior = pPrior;
2977 p->pOrderBy = 0;
2978 if( p->op==TK_UNION ){
2979 p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
2980 }
2981 sqlite3ExprDelete(db, p->pLimit);
2982 p->pLimit = pLimit;
2983 p->iLimit = 0;
2984 p->iOffset = 0;
2985
2986 /* Convert the data in the temporary table into whatever form
2987 ** it is that we currently need.
2988 */
2989 assert( unionTab==dest.iSDParm || dest.eDest!=priorOp );
2990 assert( p->pEList || db->mallocFailed );
2991 if( dest.eDest!=priorOp && db->mallocFailed==0 ){
2992 int iCont, iBreak, iStart;
2993 iBreak = sqlite3VdbeMakeLabel(pParse);
2994 iCont = sqlite3VdbeMakeLabel(pParse);
2995 computeLimitRegisters(pParse, p, iBreak);
2996 sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v);
2997 iStart = sqlite3VdbeCurrentAddr(v);
2998 selectInnerLoop(pParse, p, unionTab,
2999 0, 0, &dest, iCont, iBreak);
3000 sqlite3VdbeResolveLabel(v, iCont);
3001 sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v);
3002 sqlite3VdbeResolveLabel(v, iBreak);
3003 sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0);
3004 }
3005 break;
3006 }
3007 default: assert( p->op==TK_INTERSECT ); {
3008 int tab1, tab2;
3009 int iCont, iBreak, iStart;
3010 Expr *pLimit;
3011 int addr;
3012 SelectDest intersectdest;
3013 int r1;
3014
3015 /* INTERSECT is different from the others since it requires
3016 ** two temporary tables. Hence it has its own case. Begin
3017 ** by allocating the tables we will need.
3018 */
3019 tab1 = pParse->nTab++;
3020 tab2 = pParse->nTab++;
3021 assert( p->pOrderBy==0 );
3022
3023 addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0);
3024 assert( p->addrOpenEphm[0] == -1 );
3025 p->addrOpenEphm[0] = addr;
3026 findRightmost(p)->selFlags |= SF_UsesEphemeral;
3027 assert( p->pEList );
3028
3029 /* Code the SELECTs to our left into temporary table "tab1".
3030 */
3031 sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1);
3032 SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT left...\n"));
3033 rc = sqlite3Select(pParse, pPrior, &intersectdest);
3034 if( rc ){
3035 goto multi_select_end;
3036 }
3037
3038 /* Code the current SELECT into temporary table "tab2"
3039 */
3040 addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0);
3041 assert( p->addrOpenEphm[1] == -1 );
3042 p->addrOpenEphm[1] = addr;
3043 p->pPrior = 0;
3044 pLimit = p->pLimit;
3045 p->pLimit = 0;
3046 intersectdest.iSDParm = tab2;
3047 ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE",
3048 sqlite3SelectOpName(p->op)));
3049 SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT right...\n"));
3050 rc = sqlite3Select(pParse, p, &intersectdest);
3051 testcase( rc!=SQLITE_OK );
3052 pDelete = p->pPrior;
3053 p->pPrior = pPrior;
3054 if( p->nSelectRow>pPrior->nSelectRow ){
3055 p->nSelectRow = pPrior->nSelectRow;
3056 }
3057 sqlite3ExprDelete(db, p->pLimit);
3058 p->pLimit = pLimit;
3059
3060 /* Generate code to take the intersection of the two temporary
3061 ** tables.
3062 */
3063 if( rc ) break;
3064 assert( p->pEList );
3065 iBreak = sqlite3VdbeMakeLabel(pParse);
3066 iCont = sqlite3VdbeMakeLabel(pParse);
3067 computeLimitRegisters(pParse, p, iBreak);
3068 sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v);
3069 r1 = sqlite3GetTempReg(pParse);
3070 iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1);
3071 sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0);
3072 VdbeCoverage(v);
3073 sqlite3ReleaseTempReg(pParse, r1);
3074 selectInnerLoop(pParse, p, tab1,
3075 0, 0, &dest, iCont, iBreak);
3076 sqlite3VdbeResolveLabel(v, iCont);
3077 sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v);
3078 sqlite3VdbeResolveLabel(v, iBreak);
3079 sqlite3VdbeAddOp2(v, OP_Close, tab2, 0);
3080 sqlite3VdbeAddOp2(v, OP_Close, tab1, 0);
3081 break;
3082 }
3083 }
3084
3085 #ifndef SQLITE_OMIT_EXPLAIN
3086 if( p->pNext==0 ){
3087 ExplainQueryPlanPop(pParse);
3088 }
3089 #endif
3090 }
3091 if( pParse->nErr ) goto multi_select_end;
3092
3093 /* Compute collating sequences used by
3094 ** temporary tables needed to implement the compound select.
3095 ** Attach the KeyInfo structure to all temporary tables.
3096 **
3097 ** This section is run by the right-most SELECT statement only.
3098 ** SELECT statements to the left always skip this part. The right-most
3099 ** SELECT might also skip this part if it has no ORDER BY clause and
3100 ** no temp tables are required.
3101 */
3102 if( p->selFlags & SF_UsesEphemeral ){
3103 int i; /* Loop counter */
3104 KeyInfo *pKeyInfo; /* Collating sequence for the result set */
3105 Select *pLoop; /* For looping through SELECT statements */
3106 CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */
3107 int nCol; /* Number of columns in result set */
3108
3109 assert( p->pNext==0 );
3110 assert( p->pEList!=0 );
3111 nCol = p->pEList->nExpr;
3112 pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1);
3113 if( !pKeyInfo ){
3114 rc = SQLITE_NOMEM_BKPT;
3115 goto multi_select_end;
3116 }
3117 for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){
3118 *apColl = multiSelectCollSeq(pParse, p, i);
3119 if( 0==*apColl ){
3120 *apColl = db->pDfltColl;
3121 }
3122 }
3123
3124 for(pLoop=p; pLoop; pLoop=pLoop->pPrior){
3125 for(i=0; i<2; i++){
3126 int addr = pLoop->addrOpenEphm[i];
3127 if( addr<0 ){
3128 /* If [0] is unused then [1] is also unused. So we can
3129 ** always safely abort as soon as the first unused slot is found */
3130 assert( pLoop->addrOpenEphm[1]<0 );
3131 break;
3132 }
3133 sqlite3VdbeChangeP2(v, addr, nCol);
3134 sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo),
3135 P4_KEYINFO);
3136 pLoop->addrOpenEphm[i] = -1;
3137 }
3138 }
3139 sqlite3KeyInfoUnref(pKeyInfo);
3140 }
3141
3142multi_select_end:
3143 pDest->iSdst = dest.iSdst;
3144 pDest->nSdst = dest.nSdst;
3145 if( pDelete ){
3146 sqlite3ParserAddCleanup(pParse,
3147 (void(*)(sqlite3*,void*))sqlite3SelectDelete,
3148 pDelete);
3149 }
3150 return rc;
3151}
3152#endif /* SQLITE_OMIT_COMPOUND_SELECT */
3153
3154/*
3155** Error message for when two or more terms of a compound select have different
3156** size result sets.
3157*/
3158void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){
3159 if( p->selFlags & SF_Values ){
3160 sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms");
3161 }else{
3162 sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s"
3163 " do not have the same number of result columns",
3164 sqlite3SelectOpName(p->op));
3165 }
3166}
3167
3168/*
3169** Code an output subroutine for a coroutine implementation of a
3170** SELECT statment.
3171**
3172** The data to be output is contained in pIn->iSdst. There are
3173** pIn->nSdst columns to be output. pDest is where the output should
3174** be sent.
3175**
3176** regReturn is the number of the register holding the subroutine
3177** return address.
3178**
3179** If regPrev>0 then it is the first register in a vector that
3180** records the previous output. mem[regPrev] is a flag that is false
3181** if there has been no previous output. If regPrev>0 then code is
3182** generated to suppress duplicates. pKeyInfo is used for comparing
3183** keys.
3184**
3185** If the LIMIT found in p->iLimit is reached, jump immediately to
3186** iBreak.
3187*/
3188static int generateOutputSubroutine(
3189 Parse *pParse, /* Parsing context */
3190 Select *p, /* The SELECT statement */
3191 SelectDest *pIn, /* Coroutine supplying data */
3192 SelectDest *pDest, /* Where to send the data */
3193 int regReturn, /* The return address register */
3194 int regPrev, /* Previous result register. No uniqueness if 0 */
3195 KeyInfo *pKeyInfo, /* For comparing with previous entry */
3196 int iBreak /* Jump here if we hit the LIMIT */
3197){
3198 Vdbe *v = pParse->pVdbe;
3199 int iContinue;
3200 int addr;
3201
3202 addr = sqlite3VdbeCurrentAddr(v);
3203 iContinue = sqlite3VdbeMakeLabel(pParse);
3204
3205 /* Suppress duplicates for UNION, EXCEPT, and INTERSECT
3206 */
3207 if( regPrev ){
3208 int addr1, addr2;
3209 addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v);
3210 addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst,
3211 (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO);
3212 sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v);
3213 sqlite3VdbeJumpHere(v, addr1);
3214 sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1);
3215 sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev);
3216 }
3217 if( pParse->db->mallocFailed ) return 0;
3218
3219 /* Suppress the first OFFSET entries if there is an OFFSET clause
3220 */
3221 codeOffset(v, p->iOffset, iContinue);
3222
3223 assert( pDest->eDest!=SRT_Exists );
3224 assert( pDest->eDest!=SRT_Table );
3225 switch( pDest->eDest ){
3226 /* Store the result as data using a unique key.
3227 */
3228 case SRT_EphemTab: {
3229 int r1 = sqlite3GetTempReg(pParse);
3230 int r2 = sqlite3GetTempReg(pParse);
3231 sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1);
3232 sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2);
3233 sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2);
3234 sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
3235 sqlite3ReleaseTempReg(pParse, r2);
3236 sqlite3ReleaseTempReg(pParse, r1);
3237 break;
3238 }
3239
3240#ifndef SQLITE_OMIT_SUBQUERY
3241 /* If we are creating a set for an "expr IN (SELECT ...)".
3242 */
3243 case SRT_Set: {
3244 int r1;
3245 testcase( pIn->nSdst>1 );
3246 r1 = sqlite3GetTempReg(pParse);
3247 sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst,
3248 r1, pDest->zAffSdst, pIn->nSdst);
3249 sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1,
3250 pIn->iSdst, pIn->nSdst);
3251 sqlite3ReleaseTempReg(pParse, r1);
3252 break;
3253 }
3254
3255 /* If this is a scalar select that is part of an expression, then
3256 ** store the results in the appropriate memory cell and break out
3257 ** of the scan loop. Note that the select might return multiple columns
3258 ** if it is the RHS of a row-value IN operator.
3259 */
3260 case SRT_Mem: {
3261 testcase( pIn->nSdst>1 );
3262 sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, pIn->nSdst);
3263 /* The LIMIT clause will jump out of the loop for us */
3264 break;
3265 }
3266#endif /* #ifndef SQLITE_OMIT_SUBQUERY */
3267
3268 /* The results are stored in a sequence of registers
3269 ** starting at pDest->iSdst. Then the co-routine yields.
3270 */
3271 case SRT_Coroutine: {
3272 if( pDest->iSdst==0 ){
3273 pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst);
3274 pDest->nSdst = pIn->nSdst;
3275 }
3276 sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pIn->nSdst);
3277 sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm);
3278 break;
3279 }
3280
3281 /* If none of the above, then the result destination must be
3282 ** SRT_Output. This routine is never called with any other
3283 ** destination other than the ones handled above or SRT_Output.
3284 **
3285 ** For SRT_Output, results are stored in a sequence of registers.
3286 ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to
3287 ** return the next row of result.
3288 */
3289 default: {
3290 assert( pDest->eDest==SRT_Output );
3291 sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst);
3292 break;
3293 }
3294 }
3295
3296 /* Jump to the end of the loop if the LIMIT is reached.
3297 */
3298 if( p->iLimit ){
3299 sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v);
3300 }
3301
3302 /* Generate the subroutine return
3303 */
3304 sqlite3VdbeResolveLabel(v, iContinue);
3305 sqlite3VdbeAddOp1(v, OP_Return, regReturn);
3306
3307 return addr;
3308}
3309
3310/*
3311** Alternative compound select code generator for cases when there
3312** is an ORDER BY clause.
3313**
3314** We assume a query of the following form:
3315**
3316** <selectA> <operator> <selectB> ORDER BY <orderbylist>
3317**
3318** <operator> is one of UNION ALL, UNION, EXCEPT, or INTERSECT. The idea
3319** is to code both <selectA> and <selectB> with the ORDER BY clause as
3320** co-routines. Then run the co-routines in parallel and merge the results
3321** into the output. In addition to the two coroutines (called selectA and
3322** selectB) there are 7 subroutines:
3323**
3324** outA: Move the output of the selectA coroutine into the output
3325** of the compound query.
3326**
3327** outB: Move the output of the selectB coroutine into the output
3328** of the compound query. (Only generated for UNION and
3329** UNION ALL. EXCEPT and INSERTSECT never output a row that
3330** appears only in B.)
3331**
3332** AltB: Called when there is data from both coroutines and A<B.
3333**
3334** AeqB: Called when there is data from both coroutines and A==B.
3335**
3336** AgtB: Called when there is data from both coroutines and A>B.
3337**
3338** EofA: Called when data is exhausted from selectA.
3339**
3340** EofB: Called when data is exhausted from selectB.
3341**
3342** The implementation of the latter five subroutines depend on which
3343** <operator> is used:
3344**
3345**
3346** UNION ALL UNION EXCEPT INTERSECT
3347** ------------- ----------------- -------------- -----------------
3348** AltB: outA, nextA outA, nextA outA, nextA nextA
3349**
3350** AeqB: outA, nextA nextA nextA outA, nextA
3351**
3352** AgtB: outB, nextB outB, nextB nextB nextB
3353**
3354** EofA: outB, nextB outB, nextB halt halt
3355**
3356** EofB: outA, nextA outA, nextA outA, nextA halt
3357**
3358** In the AltB, AeqB, and AgtB subroutines, an EOF on A following nextA
3359** causes an immediate jump to EofA and an EOF on B following nextB causes
3360** an immediate jump to EofB. Within EofA and EofB, and EOF on entry or
3361** following nextX causes a jump to the end of the select processing.
3362**
3363** Duplicate removal in the UNION, EXCEPT, and INTERSECT cases is handled
3364** within the output subroutine. The regPrev register set holds the previously
3365** output value. A comparison is made against this value and the output
3366** is skipped if the next results would be the same as the previous.
3367**
3368** The implementation plan is to implement the two coroutines and seven
3369** subroutines first, then put the control logic at the bottom. Like this:
3370**
3371** goto Init
3372** coA: coroutine for left query (A)
3373** coB: coroutine for right query (B)
3374** outA: output one row of A
3375** outB: output one row of B (UNION and UNION ALL only)
3376** EofA: ...
3377** EofB: ...
3378** AltB: ...
3379** AeqB: ...
3380** AgtB: ...
3381** Init: initialize coroutine registers
3382** yield coA
3383** if eof(A) goto EofA
3384** yield coB
3385** if eof(B) goto EofB
3386** Cmpr: Compare A, B
3387** Jump AltB, AeqB, AgtB
3388** End: ...
3389**
3390** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not
3391** actually called using Gosub and they do not Return. EofA and EofB loop
3392** until all data is exhausted then jump to the "end" labe. AltB, AeqB,
3393** and AgtB jump to either L2 or to one of EofA or EofB.
3394*/
3395#ifndef SQLITE_OMIT_COMPOUND_SELECT
3396static int multiSelectOrderBy(
3397 Parse *pParse, /* Parsing context */
3398 Select *p, /* The right-most of SELECTs to be coded */
3399 SelectDest *pDest /* What to do with query results */
3400){
3401 int i, j; /* Loop counters */
3402 Select *pPrior; /* Another SELECT immediately to our left */
3403 Select *pSplit; /* Left-most SELECT in the right-hand group */
3404 int nSelect; /* Number of SELECT statements in the compound */
3405 Vdbe *v; /* Generate code to this VDBE */
3406 SelectDest destA; /* Destination for coroutine A */
3407 SelectDest destB; /* Destination for coroutine B */
3408 int regAddrA; /* Address register for select-A coroutine */
3409 int regAddrB; /* Address register for select-B coroutine */
3410 int addrSelectA; /* Address of the select-A coroutine */
3411 int addrSelectB; /* Address of the select-B coroutine */
3412 int regOutA; /* Address register for the output-A subroutine */
3413 int regOutB; /* Address register for the output-B subroutine */
3414 int addrOutA; /* Address of the output-A subroutine */
3415 int addrOutB = 0; /* Address of the output-B subroutine */
3416 int addrEofA; /* Address of the select-A-exhausted subroutine */
3417 int addrEofA_noB; /* Alternate addrEofA if B is uninitialized */
3418 int addrEofB; /* Address of the select-B-exhausted subroutine */
3419 int addrAltB; /* Address of the A<B subroutine */
3420 int addrAeqB; /* Address of the A==B subroutine */
3421 int addrAgtB; /* Address of the A>B subroutine */
3422 int regLimitA; /* Limit register for select-A */
3423 int regLimitB; /* Limit register for select-A */
3424 int regPrev; /* A range of registers to hold previous output */
3425 int savedLimit; /* Saved value of p->iLimit */
3426 int savedOffset; /* Saved value of p->iOffset */
3427 int labelCmpr; /* Label for the start of the merge algorithm */
3428 int labelEnd; /* Label for the end of the overall SELECT stmt */
3429 int addr1; /* Jump instructions that get retargetted */
3430 int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */
3431 KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */
3432 KeyInfo *pKeyMerge; /* Comparison information for merging rows */
3433 sqlite3 *db; /* Database connection */
3434 ExprList *pOrderBy; /* The ORDER BY clause */
3435 int nOrderBy; /* Number of terms in the ORDER BY clause */
3436 u32 *aPermute; /* Mapping from ORDER BY terms to result set columns */
3437
3438 assert( p->pOrderBy!=0 );
3439 assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */
3440 db = pParse->db;
3441 v = pParse->pVdbe;
3442 assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */
3443 labelEnd = sqlite3VdbeMakeLabel(pParse);
3444 labelCmpr = sqlite3VdbeMakeLabel(pParse);
3445
3446
3447 /* Patch up the ORDER BY clause
3448 */
3449 op = p->op;
3450 assert( p->pPrior->pOrderBy==0 );
3451 pOrderBy = p->pOrderBy;
3452 assert( pOrderBy );
3453 nOrderBy = pOrderBy->nExpr;
3454
3455 /* For operators other than UNION ALL we have to make sure that
3456 ** the ORDER BY clause covers every term of the result set. Add
3457 ** terms to the ORDER BY clause as necessary.
3458 */
3459 if( op!=TK_ALL ){
3460 for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){
3461 struct ExprList_item *pItem;
3462 for(j=0, pItem=pOrderBy->a; j<nOrderBy; j++, pItem++){
3463 assert( pItem!=0 );
3464 assert( pItem->u.x.iOrderByCol>0 );
3465 if( pItem->u.x.iOrderByCol==i ) break;
3466 }
3467 if( j==nOrderBy ){
3468 Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0);
3469 if( pNew==0 ) return SQLITE_NOMEM_BKPT;
3470 pNew->flags |= EP_IntValue;
3471 pNew->u.iValue = i;
3472 p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew);
3473 if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i;
3474 }
3475 }
3476 }
3477
3478 /* Compute the comparison permutation and keyinfo that is used with
3479 ** the permutation used to determine if the next
3480 ** row of results comes from selectA or selectB. Also add explicit
3481 ** collations to the ORDER BY clause terms so that when the subqueries
3482 ** to the right and the left are evaluated, they use the correct
3483 ** collation.
3484 */
3485 aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1));
3486 if( aPermute ){
3487 struct ExprList_item *pItem;
3488 aPermute[0] = nOrderBy;
3489 for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){
3490 assert( pItem!=0 );
3491 assert( pItem->u.x.iOrderByCol>0 );
3492 assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr );
3493 aPermute[i] = pItem->u.x.iOrderByCol - 1;
3494 }
3495 pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1);
3496 }else{
3497 pKeyMerge = 0;
3498 }
3499
3500 /* Allocate a range of temporary registers and the KeyInfo needed
3501 ** for the logic that removes duplicate result rows when the
3502 ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL).
3503 */
3504 if( op==TK_ALL ){
3505 regPrev = 0;
3506 }else{
3507 int nExpr = p->pEList->nExpr;
3508 assert( nOrderBy>=nExpr || db->mallocFailed );
3509 regPrev = pParse->nMem+1;
3510 pParse->nMem += nExpr+1;
3511 sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev);
3512 pKeyDup = sqlite3KeyInfoAlloc(db, nExpr, 1);
3513 if( pKeyDup ){
3514 assert( sqlite3KeyInfoIsWriteable(pKeyDup) );
3515 for(i=0; i<nExpr; i++){
3516 pKeyDup->aColl[i] = multiSelectCollSeq(pParse, p, i);
3517 pKeyDup->aSortFlags[i] = 0;
3518 }
3519 }
3520 }
3521
3522 /* Separate the left and the right query from one another
3523 */
3524 nSelect = 1;
3525 if( (op==TK_ALL || op==TK_UNION)
3526 && OptimizationEnabled(db, SQLITE_BalancedMerge)
3527 ){
3528 for(pSplit=p; pSplit->pPrior!=0 && pSplit->op==op; pSplit=pSplit->pPrior){
3529 nSelect++;
3530 assert( pSplit->pPrior->pNext==pSplit );
3531 }
3532 }
3533 if( nSelect<=3 ){
3534 pSplit = p;
3535 }else{
3536 pSplit = p;
3537 for(i=2; i<nSelect; i+=2){ pSplit = pSplit->pPrior; }
3538 }
3539 pPrior = pSplit->pPrior;
3540 assert( pPrior!=0 );
3541 pSplit->pPrior = 0;
3542 pPrior->pNext = 0;
3543 assert( p->pOrderBy == pOrderBy );
3544 assert( pOrderBy!=0 || db->mallocFailed );
3545 pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0);
3546 sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER");
3547 sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER");
3548
3549 /* Compute the limit registers */
3550 computeLimitRegisters(pParse, p, labelEnd);
3551 if( p->iLimit && op==TK_ALL ){
3552 regLimitA = ++pParse->nMem;
3553 regLimitB = ++pParse->nMem;
3554 sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit,
3555 regLimitA);
3556 sqlite3VdbeAddOp2(v, OP_Copy, regLimitA, regLimitB);
3557 }else{
3558 regLimitA = regLimitB = 0;
3559 }
3560 sqlite3ExprDelete(db, p->pLimit);
3561 p->pLimit = 0;
3562
3563 regAddrA = ++pParse->nMem;
3564 regAddrB = ++pParse->nMem;
3565 regOutA = ++pParse->nMem;
3566 regOutB = ++pParse->nMem;
3567 sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA);
3568 sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB);
3569
3570 ExplainQueryPlan((pParse, 1, "MERGE (%s)", sqlite3SelectOpName(p->op)));
3571
3572 /* Generate a coroutine to evaluate the SELECT statement to the
3573 ** left of the compound operator - the "A" select.
3574 */
3575 addrSelectA = sqlite3VdbeCurrentAddr(v) + 1;
3576 addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA);
3577 VdbeComment((v, "left SELECT"));
3578 pPrior->iLimit = regLimitA;
3579 ExplainQueryPlan((pParse, 1, "LEFT"));
3580 sqlite3Select(pParse, pPrior, &destA);
3581 sqlite3VdbeEndCoroutine(v, regAddrA);
3582 sqlite3VdbeJumpHere(v, addr1);
3583
3584 /* Generate a coroutine to evaluate the SELECT statement on
3585 ** the right - the "B" select
3586 */
3587 addrSelectB = sqlite3VdbeCurrentAddr(v) + 1;
3588 addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB);
3589 VdbeComment((v, "right SELECT"));
3590 savedLimit = p->iLimit;
3591 savedOffset = p->iOffset;
3592 p->iLimit = regLimitB;
3593 p->iOffset = 0;
3594 ExplainQueryPlan((pParse, 1, "RIGHT"));
3595 sqlite3Select(pParse, p, &destB);
3596 p->iLimit = savedLimit;
3597 p->iOffset = savedOffset;
3598 sqlite3VdbeEndCoroutine(v, regAddrB);
3599
3600 /* Generate a subroutine that outputs the current row of the A
3601 ** select as the next output row of the compound select.
3602 */
3603 VdbeNoopComment((v, "Output routine for A"));
3604 addrOutA = generateOutputSubroutine(pParse,
3605 p, &destA, pDest, regOutA,
3606 regPrev, pKeyDup, labelEnd);
3607
3608 /* Generate a subroutine that outputs the current row of the B
3609 ** select as the next output row of the compound select.
3610 */
3611 if( op==TK_ALL || op==TK_UNION ){
3612 VdbeNoopComment((v, "Output routine for B"));
3613 addrOutB = generateOutputSubroutine(pParse,
3614 p, &destB, pDest, regOutB,
3615 regPrev, pKeyDup, labelEnd);
3616 }
3617 sqlite3KeyInfoUnref(pKeyDup);
3618
3619 /* Generate a subroutine to run when the results from select A
3620 ** are exhausted and only data in select B remains.
3621 */
3622 if( op==TK_EXCEPT || op==TK_INTERSECT ){
3623 addrEofA_noB = addrEofA = labelEnd;
3624 }else{
3625 VdbeNoopComment((v, "eof-A subroutine"));
3626 addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB);
3627 addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd);
3628 VdbeCoverage(v);
3629 sqlite3VdbeGoto(v, addrEofA);
3630 p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
3631 }
3632
3633 /* Generate a subroutine to run when the results from select B
3634 ** are exhausted and only data in select A remains.
3635 */
3636 if( op==TK_INTERSECT ){
3637 addrEofB = addrEofA;
3638 if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow;
3639 }else{
3640 VdbeNoopComment((v, "eof-B subroutine"));
3641 addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA);
3642 sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v);
3643 sqlite3VdbeGoto(v, addrEofB);
3644 }
3645
3646 /* Generate code to handle the case of A<B
3647 */
3648 VdbeNoopComment((v, "A-lt-B subroutine"));
3649 addrAltB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA);
3650 sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v);
3651 sqlite3VdbeGoto(v, labelCmpr);
3652
3653 /* Generate code to handle the case of A==B
3654 */
3655 if( op==TK_ALL ){
3656 addrAeqB = addrAltB;
3657 }else if( op==TK_INTERSECT ){
3658 addrAeqB = addrAltB;
3659 addrAltB++;
3660 }else{
3661 VdbeNoopComment((v, "A-eq-B subroutine"));
3662 addrAeqB =
3663 sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v);
3664 sqlite3VdbeGoto(v, labelCmpr);
3665 }
3666
3667 /* Generate code to handle the case of A>B
3668 */
3669 VdbeNoopComment((v, "A-gt-B subroutine"));
3670 addrAgtB = sqlite3VdbeCurrentAddr(v);
3671 if( op==TK_ALL || op==TK_UNION ){
3672 sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB);
3673 }
3674 sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v);
3675 sqlite3VdbeGoto(v, labelCmpr);
3676
3677 /* This code runs once to initialize everything.
3678 */
3679 sqlite3VdbeJumpHere(v, addr1);
3680 sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v);
3681 sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v);
3682
3683 /* Implement the main merge loop
3684 */
3685 sqlite3VdbeResolveLabel(v, labelCmpr);
3686 sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY);
3687 sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy,
3688 (char*)pKeyMerge, P4_KEYINFO);
3689 sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE);
3690 sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v);
3691
3692 /* Jump to the this point in order to terminate the query.
3693 */
3694 sqlite3VdbeResolveLabel(v, labelEnd);
3695
3696 /* Make arrangements to free the 2nd and subsequent arms of the compound
3697 ** after the parse has finished */
3698 if( pSplit->pPrior ){
3699 sqlite3ParserAddCleanup(pParse,
3700 (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior);
3701 }
3702 pSplit->pPrior = pPrior;
3703 pPrior->pNext = pSplit;
3704 sqlite3ExprListDelete(db, pPrior->pOrderBy);
3705 pPrior->pOrderBy = 0;
3706
3707 /*** TBD: Insert subroutine calls to close cursors on incomplete
3708 **** subqueries ****/
3709 ExplainQueryPlanPop(pParse);
3710 return pParse->nErr!=0;
3711}
3712#endif
3713
3714#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
3715
3716/* An instance of the SubstContext object describes an substitution edit
3717** to be performed on a parse tree.
3718**
3719** All references to columns in table iTable are to be replaced by corresponding
3720** expressions in pEList.
3721**
3722** ## About "isOuterJoin":
3723**
3724** The isOuterJoin column indicates that the replacement will occur into a
3725** position in the parent that NULL-able due to an OUTER JOIN. Either the
3726** target slot in the parent is the right operand of a LEFT JOIN, or one of
3727** the left operands of a RIGHT JOIN. In either case, we need to potentially
3728** bypass the substituted expression with OP_IfNullRow.
3729**
3730** Suppose the original expression is an integer constant. Even though the table
3731** has the nullRow flag set, because the expression is an integer constant,
3732** it will not be NULLed out. So instead, we insert an OP_IfNullRow opcode
3733** that checks to see if the nullRow flag is set on the table. If the nullRow
3734** flag is set, then the value in the register is set to NULL and the original
3735** expression is bypassed. If the nullRow flag is not set, then the original
3736** expression runs to populate the register.
3737**
3738** Example where this is needed:
3739**
3740** CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT);
3741** CREATE TABLE t2(x INT UNIQUE);
3742**
3743** SELECT a,b,m,x FROM t1 LEFT JOIN (SELECT 59 AS m,x FROM t2) ON b=x;
3744**
3745** When the subquery on the right side of the LEFT JOIN is flattened, we
3746** have to add OP_IfNullRow in front of the OP_Integer that implements the
3747** "m" value of the subquery so that a NULL will be loaded instead of 59
3748** when processing a non-matched row of the left.
3749*/
3750typedef struct SubstContext {
3751 Parse *pParse; /* The parsing context */
3752 int iTable; /* Replace references to this table */
3753 int iNewTable; /* New table number */
3754 int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */
3755 ExprList *pEList; /* Replacement expressions */
3756 ExprList *pCList; /* Collation sequences for replacement expr */
3757} SubstContext;
3758
3759/* Forward Declarations */
3760static void substExprList(SubstContext*, ExprList*);
3761static void substSelect(SubstContext*, Select*, int);
3762
3763/*
3764** Scan through the expression pExpr. Replace every reference to
3765** a column in table number iTable with a copy of the iColumn-th
3766** entry in pEList. (But leave references to the ROWID column
3767** unchanged.)
3768**
3769** This routine is part of the flattening procedure. A subquery
3770** whose result set is defined by pEList appears as entry in the
3771** FROM clause of a SELECT such that the VDBE cursor assigned to that
3772** FORM clause entry is iTable. This routine makes the necessary
3773** changes to pExpr so that it refers directly to the source table
3774** of the subquery rather the result set of the subquery.
3775*/
3776static Expr *substExpr(
3777 SubstContext *pSubst, /* Description of the substitution */
3778 Expr *pExpr /* Expr in which substitution occurs */
3779){
3780 if( pExpr==0 ) return 0;
3781 if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON)
3782 && pExpr->w.iJoin==pSubst->iTable
3783 ){
3784 testcase( ExprHasProperty(pExpr, EP_InnerON) );
3785 pExpr->w.iJoin = pSubst->iNewTable;
3786 }
3787 if( pExpr->op==TK_COLUMN
3788 && pExpr->iTable==pSubst->iTable
3789 && !ExprHasProperty(pExpr, EP_FixedCol)
3790 ){
3791#ifdef SQLITE_ALLOW_ROWID_IN_VIEW
3792 if( pExpr->iColumn<0 ){
3793 pExpr->op = TK_NULL;
3794 }else
3795#endif
3796 {
3797 Expr *pNew;
3798 int iColumn = pExpr->iColumn;
3799 Expr *pCopy = pSubst->pEList->a[iColumn].pExpr;
3800 Expr ifNullRow;
3801 assert( pSubst->pEList!=0 && iColumn<pSubst->pEList->nExpr );
3802 assert( pExpr->pRight==0 );
3803 if( sqlite3ExprIsVector(pCopy) ){
3804 sqlite3VectorErrorMsg(pSubst->pParse, pCopy);
3805 }else{
3806 sqlite3 *db = pSubst->pParse->db;
3807 if( pSubst->isOuterJoin && pCopy->op!=TK_COLUMN ){
3808 memset(&ifNullRow, 0, sizeof(ifNullRow));
3809 ifNullRow.op = TK_IF_NULL_ROW;
3810 ifNullRow.pLeft = pCopy;
3811 ifNullRow.iTable = pSubst->iNewTable;
3812 ifNullRow.iColumn = -99;
3813 ifNullRow.flags = EP_IfNullRow;
3814 pCopy = &ifNullRow;
3815 }
3816 testcase( ExprHasProperty(pCopy, EP_Subquery) );
3817 pNew = sqlite3ExprDup(db, pCopy, 0);
3818 if( db->mallocFailed ){
3819 sqlite3ExprDelete(db, pNew);
3820 return pExpr;
3821 }
3822 if( pSubst->isOuterJoin ){
3823 ExprSetProperty(pNew, EP_CanBeNull);
3824 }
3825 if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){
3826 sqlite3SetJoinExpr(pNew, pExpr->w.iJoin,
3827 pExpr->flags & (EP_OuterON|EP_InnerON));
3828 }
3829 sqlite3ExprDelete(db, pExpr);
3830 pExpr = pNew;
3831 if( pExpr->op==TK_TRUEFALSE ){
3832 pExpr->u.iValue = sqlite3ExprTruthValue(pExpr);
3833 pExpr->op = TK_INTEGER;
3834 ExprSetProperty(pExpr, EP_IntValue);
3835 }
3836
3837 /* Ensure that the expression now has an implicit collation sequence,
3838 ** just as it did when it was a column of a view or sub-query. */
3839 {
3840 CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr);
3841 CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse,
3842 pSubst->pCList->a[iColumn].pExpr
3843 );
3844 if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){
3845 pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr,
3846 (pColl ? pColl->zName : "BINARY")
3847 );
3848 }
3849 }
3850 ExprClearProperty(pExpr, EP_Collate);
3851 }
3852 }
3853 }else{
3854 if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){
3855 pExpr->iTable = pSubst->iNewTable;
3856 }
3857 pExpr->pLeft = substExpr(pSubst, pExpr->pLeft);
3858 pExpr->pRight = substExpr(pSubst, pExpr->pRight);
3859 if( ExprUseXSelect(pExpr) ){
3860 substSelect(pSubst, pExpr->x.pSelect, 1);
3861 }else{
3862 substExprList(pSubst, pExpr->x.pList);
3863 }
3864#ifndef SQLITE_OMIT_WINDOWFUNC
3865 if( ExprHasProperty(pExpr, EP_WinFunc) ){
3866 Window *pWin = pExpr->y.pWin;
3867 pWin->pFilter = substExpr(pSubst, pWin->pFilter);
3868 substExprList(pSubst, pWin->pPartition);
3869 substExprList(pSubst, pWin->pOrderBy);
3870 }
3871#endif
3872 }
3873 return pExpr;
3874}
3875static void substExprList(
3876 SubstContext *pSubst, /* Description of the substitution */
3877 ExprList *pList /* List to scan and in which to make substitutes */
3878){
3879 int i;
3880 if( pList==0 ) return;
3881 for(i=0; i<pList->nExpr; i++){
3882 pList->a[i].pExpr = substExpr(pSubst, pList->a[i].pExpr);
3883 }
3884}
3885static void substSelect(
3886 SubstContext *pSubst, /* Description of the substitution */
3887 Select *p, /* SELECT statement in which to make substitutions */
3888 int doPrior /* Do substitutes on p->pPrior too */
3889){
3890 SrcList *pSrc;
3891 SrcItem *pItem;
3892 int i;
3893 if( !p ) return;
3894 do{
3895 substExprList(pSubst, p->pEList);
3896 substExprList(pSubst, p->pGroupBy);
3897 substExprList(pSubst, p->pOrderBy);
3898 p->pHaving = substExpr(pSubst, p->pHaving);
3899 p->pWhere = substExpr(pSubst, p->pWhere);
3900 pSrc = p->pSrc;
3901 assert( pSrc!=0 );
3902 for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){
3903 substSelect(pSubst, pItem->pSelect, 1);
3904 if( pItem->fg.isTabFunc ){
3905 substExprList(pSubst, pItem->u1.pFuncArg);
3906 }
3907 }
3908 }while( doPrior && (p = p->pPrior)!=0 );
3909}
3910#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
3911
3912#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
3913/*
3914** pSelect is a SELECT statement and pSrcItem is one item in the FROM
3915** clause of that SELECT.
3916**
3917** This routine scans the entire SELECT statement and recomputes the
3918** pSrcItem->colUsed mask.
3919*/
3920static int recomputeColumnsUsedExpr(Walker *pWalker, Expr *pExpr){
3921 SrcItem *pItem;
3922 if( pExpr->op!=TK_COLUMN ) return WRC_Continue;
3923 pItem = pWalker->u.pSrcItem;
3924 if( pItem->iCursor!=pExpr->iTable ) return WRC_Continue;
3925 if( pExpr->iColumn<0 ) return WRC_Continue;
3926 pItem->colUsed |= sqlite3ExprColUsed(pExpr);
3927 return WRC_Continue;
3928}
3929static void recomputeColumnsUsed(
3930 Select *pSelect, /* The complete SELECT statement */
3931 SrcItem *pSrcItem /* Which FROM clause item to recompute */
3932){
3933 Walker w;
3934 if( NEVER(pSrcItem->pTab==0) ) return;
3935 memset(&w, 0, sizeof(w));
3936 w.xExprCallback = recomputeColumnsUsedExpr;
3937 w.xSelectCallback = sqlite3SelectWalkNoop;
3938 w.u.pSrcItem = pSrcItem;
3939 pSrcItem->colUsed = 0;
3940 sqlite3WalkSelect(&w, pSelect);
3941}
3942#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
3943
3944#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
3945/*
3946** Assign new cursor numbers to each of the items in pSrc. For each
3947** new cursor number assigned, set an entry in the aCsrMap[] array
3948** to map the old cursor number to the new:
3949**
3950** aCsrMap[iOld+1] = iNew;
3951**
3952** The array is guaranteed by the caller to be large enough for all
3953** existing cursor numbers in pSrc. aCsrMap[0] is the array size.
3954**
3955** If pSrc contains any sub-selects, call this routine recursively
3956** on the FROM clause of each such sub-select, with iExcept set to -1.
3957*/
3958static void srclistRenumberCursors(
3959 Parse *pParse, /* Parse context */
3960 int *aCsrMap, /* Array to store cursor mappings in */
3961 SrcList *pSrc, /* FROM clause to renumber */
3962 int iExcept /* FROM clause item to skip */
3963){
3964 int i;
3965 SrcItem *pItem;
3966 for(i=0, pItem=pSrc->a; i<pSrc->nSrc; i++, pItem++){
3967 if( i!=iExcept ){
3968 Select *p;
3969 assert( pItem->iCursor < aCsrMap[0] );
3970 if( !pItem->fg.isRecursive || aCsrMap[pItem->iCursor+1]==0 ){
3971 aCsrMap[pItem->iCursor+1] = pParse->nTab++;
3972 }
3973 pItem->iCursor = aCsrMap[pItem->iCursor+1];
3974 for(p=pItem->pSelect; p; p=p->pPrior){
3975 srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1);
3976 }
3977 }
3978 }
3979}
3980
3981/*
3982** *piCursor is a cursor number. Change it if it needs to be mapped.
3983*/
3984static void renumberCursorDoMapping(Walker *pWalker, int *piCursor){
3985 int *aCsrMap = pWalker->u.aiCol;
3986 int iCsr = *piCursor;
3987 if( iCsr < aCsrMap[0] && aCsrMap[iCsr+1]>0 ){
3988 *piCursor = aCsrMap[iCsr+1];
3989 }
3990}
3991
3992/*
3993** Expression walker callback used by renumberCursors() to update
3994** Expr objects to match newly assigned cursor numbers.
3995*/
3996static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){
3997 int op = pExpr->op;
3998 if( op==TK_COLUMN || op==TK_IF_NULL_ROW ){
3999 renumberCursorDoMapping(pWalker, &pExpr->iTable);
4000 }
4001 if( ExprHasProperty(pExpr, EP_OuterON) ){
4002 renumberCursorDoMapping(pWalker, &pExpr->w.iJoin);
4003 }
4004 return WRC_Continue;
4005}
4006
4007/*
4008** Assign a new cursor number to each cursor in the FROM clause (Select.pSrc)
4009** of the SELECT statement passed as the second argument, and to each
4010** cursor in the FROM clause of any FROM clause sub-selects, recursively.
4011** Except, do not assign a new cursor number to the iExcept'th element in
4012** the FROM clause of (*p). Update all expressions and other references
4013** to refer to the new cursor numbers.
4014**
4015** Argument aCsrMap is an array that may be used for temporary working
4016** space. Two guarantees are made by the caller:
4017**
4018** * the array is larger than the largest cursor number used within the
4019** select statement passed as an argument, and
4020**
4021** * the array entries for all cursor numbers that do *not* appear in
4022** FROM clauses of the select statement as described above are
4023** initialized to zero.
4024*/
4025static void renumberCursors(
4026 Parse *pParse, /* Parse context */
4027 Select *p, /* Select to renumber cursors within */
4028 int iExcept, /* FROM clause item to skip */
4029 int *aCsrMap /* Working space */
4030){
4031 Walker w;
4032 srclistRenumberCursors(pParse, aCsrMap, p->pSrc, iExcept);
4033 memset(&w, 0, sizeof(w));
4034 w.u.aiCol = aCsrMap;
4035 w.xExprCallback = renumberCursorsCb;
4036 w.xSelectCallback = sqlite3SelectWalkNoop;
4037 sqlite3WalkSelect(&w, p);
4038}
4039#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
4040
4041/*
4042** If pSel is not part of a compound SELECT, return a pointer to its
4043** expression list. Otherwise, return a pointer to the expression list
4044** of the leftmost SELECT in the compound.
4045*/
4046static ExprList *findLeftmostExprlist(Select *pSel){
4047 while( pSel->pPrior ){
4048 pSel = pSel->pPrior;
4049 }
4050 return pSel->pEList;
4051}
4052
4053#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
4054/*
4055** This routine attempts to flatten subqueries as a performance optimization.
4056** This routine returns 1 if it makes changes and 0 if no flattening occurs.
4057**
4058** To understand the concept of flattening, consider the following
4059** query:
4060**
4061** SELECT a FROM (SELECT x+y AS a FROM t1 WHERE z<100) WHERE a>5
4062**
4063** The default way of implementing this query is to execute the
4064** subquery first and store the results in a temporary table, then
4065** run the outer query on that temporary table. This requires two
4066** passes over the data. Furthermore, because the temporary table
4067** has no indices, the WHERE clause on the outer query cannot be
4068** optimized.
4069**
4070** This routine attempts to rewrite queries such as the above into
4071** a single flat select, like this:
4072**
4073** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5
4074**
4075** The code generated for this simplification gives the same result
4076** but only has to scan the data once. And because indices might
4077** exist on the table t1, a complete scan of the data might be
4078** avoided.
4079**
4080** Flattening is subject to the following constraints:
4081**
4082** (**) We no longer attempt to flatten aggregate subqueries. Was:
4083** The subquery and the outer query cannot both be aggregates.
4084**
4085** (**) We no longer attempt to flatten aggregate subqueries. Was:
4086** (2) If the subquery is an aggregate then
4087** (2a) the outer query must not be a join and
4088** (2b) the outer query must not use subqueries
4089** other than the one FROM-clause subquery that is a candidate
4090** for flattening. (This is due to ticket [2f7170d73bf9abf80]
4091** from 2015-02-09.)
4092**
4093** (3) If the subquery is the right operand of a LEFT JOIN then
4094** (3a) the subquery may not be a join and
4095** (3b) the FROM clause of the subquery may not contain a virtual
4096** table and
4097** (**) Was: "The outer query may not have a GROUP BY." This case
4098** is now managed correctly
4099** (3d) the outer query may not be DISTINCT.
4100** See also (26) for restrictions on RIGHT JOIN.
4101**
4102** (4) The subquery can not be DISTINCT.
4103**
4104** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT
4105** sub-queries that were excluded from this optimization. Restriction
4106** (4) has since been expanded to exclude all DISTINCT subqueries.
4107**
4108** (**) We no longer attempt to flatten aggregate subqueries. Was:
4109** If the subquery is aggregate, the outer query may not be DISTINCT.
4110**
4111** (7) The subquery must have a FROM clause. TODO: For subqueries without
4112** A FROM clause, consider adding a FROM clause with the special
4113** table sqlite_once that consists of a single row containing a
4114** single NULL.
4115**
4116** (8) If the subquery uses LIMIT then the outer query may not be a join.
4117**
4118** (9) If the subquery uses LIMIT then the outer query may not be aggregate.
4119**
4120** (**) Restriction (10) was removed from the code on 2005-02-05 but we
4121** accidently carried the comment forward until 2014-09-15. Original
4122** constraint: "If the subquery is aggregate then the outer query
4123** may not use LIMIT."
4124**
4125** (11) The subquery and the outer query may not both have ORDER BY clauses.
4126**
4127** (**) Not implemented. Subsumed into restriction (3). Was previously
4128** a separate restriction deriving from ticket #350.
4129**
4130** (13) The subquery and outer query may not both use LIMIT.
4131**
4132** (14) The subquery may not use OFFSET.
4133**
4134** (15) If the outer query is part of a compound select, then the
4135** subquery may not use LIMIT.
4136** (See ticket #2339 and ticket [02a8e81d44]).
4137**
4138** (16) If the outer query is aggregate, then the subquery may not
4139** use ORDER BY. (Ticket #2942) This used to not matter
4140** until we introduced the group_concat() function.
4141**
4142** (17) If the subquery is a compound select, then
4143** (17a) all compound operators must be a UNION ALL, and
4144** (17b) no terms within the subquery compound may be aggregate
4145** or DISTINCT, and
4146** (17c) every term within the subquery compound must have a FROM clause
4147** (17d) the outer query may not be
4148** (17d1) aggregate, or
4149** (17d2) DISTINCT
4150** (17e) the subquery may not contain window functions, and
4151** (17f) the subquery must not be the RHS of a LEFT JOIN.
4152** (17g) either the subquery is the first element of the outer
4153** query or there are no RIGHT or FULL JOINs in any arm
4154** of the subquery. (This is a duplicate of condition (27b).)
4155** (17h) The corresponding result set expressions in all arms of the
4156** compound must have the same affinity.
4157**
4158** The parent and sub-query may contain WHERE clauses. Subject to
4159** rules (11), (13) and (14), they may also contain ORDER BY,
4160** LIMIT and OFFSET clauses. The subquery cannot use any compound
4161** operator other than UNION ALL because all the other compound
4162** operators have an implied DISTINCT which is disallowed by
4163** restriction (4).
4164**
4165** Also, each component of the sub-query must return the same number
4166** of result columns. This is actually a requirement for any compound
4167** SELECT statement, but all the code here does is make sure that no
4168** such (illegal) sub-query is flattened. The caller will detect the
4169** syntax error and return a detailed message.
4170**
4171** (18) If the sub-query is a compound select, then all terms of the
4172** ORDER BY clause of the parent must be copies of a term returned
4173** by the parent query.
4174**
4175** (19) If the subquery uses LIMIT then the outer query may not
4176** have a WHERE clause.
4177**
4178** (20) If the sub-query is a compound select, then it must not use
4179** an ORDER BY clause. Ticket #3773. We could relax this constraint
4180** somewhat by saying that the terms of the ORDER BY clause must
4181** appear as unmodified result columns in the outer query. But we
4182** have other optimizations in mind to deal with that case.
4183**
4184** (21) If the subquery uses LIMIT then the outer query may not be
4185** DISTINCT. (See ticket [752e1646fc]).
4186**
4187** (22) The subquery may not be a recursive CTE.
4188**
4189** (23) If the outer query is a recursive CTE, then the sub-query may not be
4190** a compound query. This restriction is because transforming the
4191** parent to a compound query confuses the code that handles
4192** recursive queries in multiSelect().
4193**
4194** (**) We no longer attempt to flatten aggregate subqueries. Was:
4195** The subquery may not be an aggregate that uses the built-in min() or
4196** or max() functions. (Without this restriction, a query like:
4197** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily
4198** return the value X for which Y was maximal.)
4199**
4200** (25) If either the subquery or the parent query contains a window
4201** function in the select list or ORDER BY clause, flattening
4202** is not attempted.
4203**
4204** (26) The subquery may not be the right operand of a RIGHT JOIN.
4205** See also (3) for restrictions on LEFT JOIN.
4206**
4207** (27) The subquery may not contain a FULL or RIGHT JOIN unless it
4208** is the first element of the parent query. Two subcases:
4209** (27a) the subquery is not a compound query.
4210** (27b) the subquery is a compound query and the RIGHT JOIN occurs
4211** in any arm of the compound query. (See also (17g).)
4212**
4213** (28) The subquery is not a MATERIALIZED CTE.
4214**
4215**
4216** In this routine, the "p" parameter is a pointer to the outer query.
4217** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query
4218** uses aggregates.
4219**
4220** If flattening is not attempted, this routine is a no-op and returns 0.
4221** If flattening is attempted this routine returns 1.
4222**
4223** All of the expression analysis must occur on both the outer query and
4224** the subquery before this routine runs.
4225*/
4226static int flattenSubquery(
4227 Parse *pParse, /* Parsing context */
4228 Select *p, /* The parent or outer SELECT statement */
4229 int iFrom, /* Index in p->pSrc->a[] of the inner subquery */
4230 int isAgg /* True if outer SELECT uses aggregate functions */
4231){
4232 const char *zSavedAuthContext = pParse->zAuthContext;
4233 Select *pParent; /* Current UNION ALL term of the other query */
4234 Select *pSub; /* The inner query or "subquery" */
4235 Select *pSub1; /* Pointer to the rightmost select in sub-query */
4236 SrcList *pSrc; /* The FROM clause of the outer query */
4237 SrcList *pSubSrc; /* The FROM clause of the subquery */
4238 int iParent; /* VDBE cursor number of the pSub result set temp table */
4239 int iNewParent = -1;/* Replacement table for iParent */
4240 int isOuterJoin = 0; /* True if pSub is the right side of a LEFT JOIN */
4241 int i; /* Loop counter */
4242 Expr *pWhere; /* The WHERE clause */
4243 SrcItem *pSubitem; /* The subquery */
4244 sqlite3 *db = pParse->db;
4245 Walker w; /* Walker to persist agginfo data */
4246 int *aCsrMap = 0;
4247
4248 /* Check to see if flattening is permitted. Return 0 if not.
4249 */
4250 assert( p!=0 );
4251 assert( p->pPrior==0 );
4252 if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0;
4253 pSrc = p->pSrc;
4254 assert( pSrc && iFrom>=0 && iFrom<pSrc->nSrc );
4255 pSubitem = &pSrc->a[iFrom];
4256 iParent = pSubitem->iCursor;
4257 pSub = pSubitem->pSelect;
4258 assert( pSub!=0 );
4259
4260#ifndef SQLITE_OMIT_WINDOWFUNC
4261 if( p->pWin || pSub->pWin ) return 0; /* Restriction (25) */
4262#endif
4263
4264 pSubSrc = pSub->pSrc;
4265 assert( pSubSrc );
4266 /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants,
4267 ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET
4268 ** because they could be computed at compile-time. But when LIMIT and OFFSET
4269 ** became arbitrary expressions, we were forced to add restrictions (13)
4270 ** and (14). */
4271 if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */
4272 if( pSub->pLimit && pSub->pLimit->pRight ) return 0; /* Restriction (14) */
4273 if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){
4274 return 0; /* Restriction (15) */
4275 }
4276 if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */
4277 if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (4) */
4278 if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){
4279 return 0; /* Restrictions (8)(9) */
4280 }
4281 if( p->pOrderBy && pSub->pOrderBy ){
4282 return 0; /* Restriction (11) */
4283 }
4284 if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */
4285 if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */
4286 if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){
4287 return 0; /* Restriction (21) */
4288 }
4289 if( pSub->selFlags & (SF_Recursive) ){
4290 return 0; /* Restrictions (22) */
4291 }
4292
4293 /*
4294 ** If the subquery is the right operand of a LEFT JOIN, then the
4295 ** subquery may not be a join itself (3a). Example of why this is not
4296 ** allowed:
4297 **
4298 ** t1 LEFT OUTER JOIN (t2 JOIN t3)
4299 **
4300 ** If we flatten the above, we would get
4301 **
4302 ** (t1 LEFT OUTER JOIN t2) JOIN t3
4303 **
4304 ** which is not at all the same thing.
4305 **
4306 ** See also tickets #306, #350, and #3300.
4307 */
4308 if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){
4309 if( pSubSrc->nSrc>1 /* (3a) */
4310 || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */
4311 || (p->selFlags & SF_Distinct)!=0 /* (3d) */
4312 || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */
4313 ){
4314 return 0;
4315 }
4316 isOuterJoin = 1;
4317 }
4318
4319 assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */
4320 if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
4321 return 0; /* Restriction (27a) */
4322 }
4323 if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){
4324 return 0; /* (28) */
4325 }
4326
4327 /* Restriction (17): If the sub-query is a compound SELECT, then it must
4328 ** use only the UNION ALL operator. And none of the simple select queries
4329 ** that make up the compound SELECT are allowed to be aggregate or distinct
4330 ** queries.
4331 */
4332 if( pSub->pPrior ){
4333 int ii;
4334 if( pSub->pOrderBy ){
4335 return 0; /* Restriction (20) */
4336 }
4337 if( isAgg || (p->selFlags & SF_Distinct)!=0 || isOuterJoin>0 ){
4338 return 0; /* (17d1), (17d2), or (17f) */
4339 }
4340 for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){
4341 testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct );
4342 testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate );
4343 assert( pSub->pSrc!=0 );
4344 assert( (pSub->selFlags & SF_Recursive)==0 );
4345 assert( pSub->pEList->nExpr==pSub1->pEList->nExpr );
4346 if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 /* (17b) */
4347 || (pSub1->pPrior && pSub1->op!=TK_ALL) /* (17a) */
4348 || pSub1->pSrc->nSrc<1 /* (17c) */
4349#ifndef SQLITE_OMIT_WINDOWFUNC
4350 || pSub1->pWin /* (17e) */
4351#endif
4352 ){
4353 return 0;
4354 }
4355 if( iFrom>0 && (pSub1->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
4356 /* Without this restriction, the JT_LTORJ flag would end up being
4357 ** omitted on left-hand tables of the right join that is being
4358 ** flattened. */
4359 return 0; /* Restrictions (17g), (27b) */
4360 }
4361 testcase( pSub1->pSrc->nSrc>1 );
4362 }
4363
4364 /* Restriction (18). */
4365 if( p->pOrderBy ){
4366 for(ii=0; ii<p->pOrderBy->nExpr; ii++){
4367 if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0;
4368 }
4369 }
4370
4371 /* Restriction (23) */
4372 if( (p->selFlags & SF_Recursive) ) return 0;
4373
4374 /* Restriction (17h) */
4375 for(ii=0; ii<pSub->pEList->nExpr; ii++){
4376 char aff;
4377 assert( pSub->pEList->a[ii].pExpr!=0 );
4378 aff = sqlite3ExprAffinity(pSub->pEList->a[ii].pExpr);
4379 for(pSub1=pSub->pPrior; pSub1; pSub1=pSub1->pPrior){
4380 assert( pSub1->pEList!=0 );
4381 assert( pSub1->pEList->nExpr>ii );
4382 assert( pSub1->pEList->a[ii].pExpr!=0 );
4383 if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){
4384 return 0;
4385 }
4386 }
4387 }
4388
4389 if( pSrc->nSrc>1 ){
4390 if( pParse->nSelect>500 ) return 0;
4391 if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0;
4392 aCsrMap = sqlite3DbMallocZero(db, ((i64)pParse->nTab+1)*sizeof(int));
4393 if( aCsrMap ) aCsrMap[0] = pParse->nTab;
4394 }
4395 }
4396
4397 /***** If we reach this point, flattening is permitted. *****/
4398 SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n",
4399 pSub->selId, pSub, iFrom));
4400
4401 /* Authorize the subquery */
4402 pParse->zAuthContext = pSubitem->zName;
4403 TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0);
4404 testcase( i==SQLITE_DENY );
4405 pParse->zAuthContext = zSavedAuthContext;
4406
4407 /* Delete the transient structures associated with thesubquery */
4408 pSub1 = pSubitem->pSelect;
4409 sqlite3DbFree(db, pSubitem->zDatabase);
4410 sqlite3DbFree(db, pSubitem->zName);
4411 sqlite3DbFree(db, pSubitem->zAlias);
4412 pSubitem->zDatabase = 0;
4413 pSubitem->zName = 0;
4414 pSubitem->zAlias = 0;
4415 pSubitem->pSelect = 0;
4416 assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 );
4417
4418 /* If the sub-query is a compound SELECT statement, then (by restrictions
4419 ** 17 and 18 above) it must be a UNION ALL and the parent query must
4420 ** be of the form:
4421 **
4422 ** SELECT <expr-list> FROM (<sub-query>) <where-clause>
4423 **
4424 ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block
4425 ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or
4426 ** OFFSET clauses and joins them to the left-hand-side of the original
4427 ** using UNION ALL operators. In this case N is the number of simple
4428 ** select statements in the compound sub-query.
4429 **
4430 ** Example:
4431 **
4432 ** SELECT a+1 FROM (
4433 ** SELECT x FROM tab
4434 ** UNION ALL
4435 ** SELECT y FROM tab
4436 ** UNION ALL
4437 ** SELECT abs(z*2) FROM tab2
4438 ** ) WHERE a!=5 ORDER BY 1
4439 **
4440 ** Transformed into:
4441 **
4442 ** SELECT x+1 FROM tab WHERE x+1!=5
4443 ** UNION ALL
4444 ** SELECT y+1 FROM tab WHERE y+1!=5
4445 ** UNION ALL
4446 ** SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5
4447 ** ORDER BY 1
4448 **
4449 ** We call this the "compound-subquery flattening".
4450 */
4451 for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){
4452 Select *pNew;
4453 ExprList *pOrderBy = p->pOrderBy;
4454 Expr *pLimit = p->pLimit;
4455 Select *pPrior = p->pPrior;
4456 Table *pItemTab = pSubitem->pTab;
4457 pSubitem->pTab = 0;
4458 p->pOrderBy = 0;
4459 p->pPrior = 0;
4460 p->pLimit = 0;
4461 pNew = sqlite3SelectDup(db, p, 0);
4462 p->pLimit = pLimit;
4463 p->pOrderBy = pOrderBy;
4464 p->op = TK_ALL;
4465 pSubitem->pTab = pItemTab;
4466 if( pNew==0 ){
4467 p->pPrior = pPrior;
4468 }else{
4469 pNew->selId = ++pParse->nSelect;
4470 if( aCsrMap && ALWAYS(db->mallocFailed==0) ){
4471 renumberCursors(pParse, pNew, iFrom, aCsrMap);
4472 }
4473 pNew->pPrior = pPrior;
4474 if( pPrior ) pPrior->pNext = pNew;
4475 pNew->pNext = p;
4476 p->pPrior = pNew;
4477 SELECTTRACE(2,pParse,p,("compound-subquery flattener"
4478 " creates %u as peer\n",pNew->selId));
4479 }
4480 assert( pSubitem->pSelect==0 );
4481 }
4482 sqlite3DbFree(db, aCsrMap);
4483 if( db->mallocFailed ){
4484 pSubitem->pSelect = pSub1;
4485 return 1;
4486 }
4487
4488 /* Defer deleting the Table object associated with the
4489 ** subquery until code generation is
4490 ** complete, since there may still exist Expr.pTab entries that
4491 ** refer to the subquery even after flattening. Ticket #3346.
4492 **
4493 ** pSubitem->pTab is always non-NULL by test restrictions and tests above.
4494 */
4495 if( ALWAYS(pSubitem->pTab!=0) ){
4496 Table *pTabToDel = pSubitem->pTab;
4497 if( pTabToDel->nTabRef==1 ){
4498 Parse *pToplevel = sqlite3ParseToplevel(pParse);
4499 sqlite3ParserAddCleanup(pToplevel,
4500 (void(*)(sqlite3*,void*))sqlite3DeleteTable,
4501 pTabToDel);
4502 testcase( pToplevel->earlyCleanup );
4503 }else{
4504 pTabToDel->nTabRef--;
4505 }
4506 pSubitem->pTab = 0;
4507 }
4508
4509 /* The following loop runs once for each term in a compound-subquery
4510 ** flattening (as described above). If we are doing a different kind
4511 ** of flattening - a flattening other than a compound-subquery flattening -
4512 ** then this loop only runs once.
4513 **
4514 ** This loop moves all of the FROM elements of the subquery into the
4515 ** the FROM clause of the outer query. Before doing this, remember
4516 ** the cursor number for the original outer query FROM element in
4517 ** iParent. The iParent cursor will never be used. Subsequent code
4518 ** will scan expressions looking for iParent references and replace
4519 ** those references with expressions that resolve to the subquery FROM
4520 ** elements we are now copying in.
4521 */
4522 pSub = pSub1;
4523 for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){
4524 int nSubSrc;
4525 u8 jointype = 0;
4526 u8 ltorj = pSrc->a[iFrom].fg.jointype & JT_LTORJ;
4527 assert( pSub!=0 );
4528 pSubSrc = pSub->pSrc; /* FROM clause of subquery */
4529 nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */
4530 pSrc = pParent->pSrc; /* FROM clause of the outer query */
4531
4532 if( pParent==p ){
4533 jointype = pSubitem->fg.jointype; /* First time through the loop */
4534 }
4535
4536 /* The subquery uses a single slot of the FROM clause of the outer
4537 ** query. If the subquery has more than one element in its FROM clause,
4538 ** then expand the outer query to make space for it to hold all elements
4539 ** of the subquery.
4540 **
4541 ** Example:
4542 **
4543 ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB;
4544 **
4545 ** The outer query has 3 slots in its FROM clause. One slot of the
4546 ** outer query (the middle slot) is used by the subquery. The next
4547 ** block of code will expand the outer query FROM clause to 4 slots.
4548 ** The middle slot is expanded to two slots in order to make space
4549 ** for the two elements in the FROM clause of the subquery.
4550 */
4551 if( nSubSrc>1 ){
4552 pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1);
4553 if( pSrc==0 ) break;
4554 pParent->pSrc = pSrc;
4555 }
4556
4557 /* Transfer the FROM clause terms from the subquery into the
4558 ** outer query.
4559 */
4560 for(i=0; i<nSubSrc; i++){
4561 SrcItem *pItem = &pSrc->a[i+iFrom];
4562 if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing);
4563 assert( pItem->fg.isTabFunc==0 );
4564 *pItem = pSubSrc->a[i];
4565 pItem->fg.jointype |= ltorj;
4566 iNewParent = pSubSrc->a[i].iCursor;
4567 memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i]));
4568 }
4569 pSrc->a[iFrom].fg.jointype &= JT_LTORJ;
4570 pSrc->a[iFrom].fg.jointype |= jointype | ltorj;
4571
4572 /* Now begin substituting subquery result set expressions for
4573 ** references to the iParent in the outer query.
4574 **
4575 ** Example:
4576 **
4577 ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b;
4578 ** \ \_____________ subquery __________/ /
4579 ** \_____________________ outer query ______________________________/
4580 **
4581 ** We look at every expression in the outer query and every place we see
4582 ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10".
4583 */
4584 if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){
4585 /* At this point, any non-zero iOrderByCol values indicate that the
4586 ** ORDER BY column expression is identical to the iOrderByCol'th
4587 ** expression returned by SELECT statement pSub. Since these values
4588 ** do not necessarily correspond to columns in SELECT statement pParent,
4589 ** zero them before transfering the ORDER BY clause.
4590 **
4591 ** Not doing this may cause an error if a subsequent call to this
4592 ** function attempts to flatten a compound sub-query into pParent
4593 ** (the only way this can happen is if the compound sub-query is
4594 ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */
4595 ExprList *pOrderBy = pSub->pOrderBy;
4596 for(i=0; i<pOrderBy->nExpr; i++){
4597 pOrderBy->a[i].u.x.iOrderByCol = 0;
4598 }
4599 assert( pParent->pOrderBy==0 );
4600 pParent->pOrderBy = pOrderBy;
4601 pSub->pOrderBy = 0;
4602 }
4603 pWhere = pSub->pWhere;
4604 pSub->pWhere = 0;
4605 if( isOuterJoin>0 ){
4606 sqlite3SetJoinExpr(pWhere, iNewParent, EP_OuterON);
4607 }
4608 if( pWhere ){
4609 if( pParent->pWhere ){
4610 pParent->pWhere = sqlite3PExpr(pParse, TK_AND, pWhere, pParent->pWhere);
4611 }else{
4612 pParent->pWhere = pWhere;
4613 }
4614 }
4615 if( db->mallocFailed==0 ){
4616 SubstContext x;
4617 x.pParse = pParse;
4618 x.iTable = iParent;
4619 x.iNewTable = iNewParent;
4620 x.isOuterJoin = isOuterJoin;
4621 x.pEList = pSub->pEList;
4622 x.pCList = findLeftmostExprlist(pSub);
4623 substSelect(&x, pParent, 0);
4624 }
4625
4626 /* The flattened query is a compound if either the inner or the
4627 ** outer query is a compound. */
4628 pParent->selFlags |= pSub->selFlags & SF_Compound;
4629 assert( (pSub->selFlags & SF_Distinct)==0 ); /* restriction (17b) */
4630
4631 /*
4632 ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y;
4633 **
4634 ** One is tempted to try to add a and b to combine the limits. But this
4635 ** does not work if either limit is negative.
4636 */
4637 if( pSub->pLimit ){
4638 pParent->pLimit = pSub->pLimit;
4639 pSub->pLimit = 0;
4640 }
4641
4642 /* Recompute the SrcItem.colUsed masks for the flattened
4643 ** tables. */
4644 for(i=0; i<nSubSrc; i++){
4645 recomputeColumnsUsed(pParent, &pSrc->a[i+iFrom]);
4646 }
4647 }
4648
4649 /* Finially, delete what is left of the subquery and return
4650 ** success.
4651 */
4652 sqlite3AggInfoPersistWalkerInit(&w, pParse);
4653 sqlite3WalkSelect(&w,pSub1);
4654 sqlite3SelectDelete(db, pSub1);
4655
4656#if TREETRACE_ENABLED
4657 if( sqlite3TreeTrace & 0x100 ){
4658 SELECTTRACE(0x100,pParse,p,("After flattening:\n"));
4659 sqlite3TreeViewSelect(0, p, 0);
4660 }
4661#endif
4662
4663 return 1;
4664}
4665#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
4666
4667/*
4668** A structure to keep track of all of the column values that are fixed to
4669** a known value due to WHERE clause constraints of the form COLUMN=VALUE.
4670*/
4671typedef struct WhereConst WhereConst;
4672struct WhereConst {
4673 Parse *pParse; /* Parsing context */
4674 u8 *pOomFault; /* Pointer to pParse->db->mallocFailed */
4675 int nConst; /* Number for COLUMN=CONSTANT terms */
4676 int nChng; /* Number of times a constant is propagated */
4677 int bHasAffBlob; /* At least one column in apExpr[] as affinity BLOB */
4678 u32 mExcludeOn; /* Which ON expressions to exclude from considertion.
4679 ** Either EP_OuterON or EP_InnerON|EP_OuterON */
4680 Expr **apExpr; /* [i*2] is COLUMN and [i*2+1] is VALUE */
4681};
4682
4683/*
4684** Add a new entry to the pConst object. Except, do not add duplicate
4685** pColumn entires. Also, do not add if doing so would not be appropriate.
4686**
4687** The caller guarantees the pColumn is a column and pValue is a constant.
4688** This routine has to do some additional checks before completing the
4689** insert.
4690*/
4691static void constInsert(
4692 WhereConst *pConst, /* The WhereConst into which we are inserting */
4693 Expr *pColumn, /* The COLUMN part of the constraint */
4694 Expr *pValue, /* The VALUE part of the constraint */
4695 Expr *pExpr /* Overall expression: COLUMN=VALUE or VALUE=COLUMN */
4696){
4697 int i;
4698 assert( pColumn->op==TK_COLUMN );
4699 assert( sqlite3ExprIsConstant(pValue) );
4700
4701 if( ExprHasProperty(pColumn, EP_FixedCol) ) return;
4702 if( sqlite3ExprAffinity(pValue)!=0 ) return;
4703 if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pConst->pParse,pExpr)) ){
4704 return;
4705 }
4706
4707 /* 2018-10-25 ticket [cf5ed20f]
4708 ** Make sure the same pColumn is not inserted more than once */
4709 for(i=0; i<pConst->nConst; i++){
4710 const Expr *pE2 = pConst->apExpr[i*2];
4711 assert( pE2->op==TK_COLUMN );
4712 if( pE2->iTable==pColumn->iTable
4713 && pE2->iColumn==pColumn->iColumn
4714 ){
4715 return; /* Already present. Return without doing anything. */
4716 }
4717 }
4718 if( sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){
4719 pConst->bHasAffBlob = 1;
4720 }
4721
4722 pConst->nConst++;
4723 pConst->apExpr = sqlite3DbReallocOrFree(pConst->pParse->db, pConst->apExpr,
4724 pConst->nConst*2*sizeof(Expr*));
4725 if( pConst->apExpr==0 ){
4726 pConst->nConst = 0;
4727 }else{
4728 pConst->apExpr[pConst->nConst*2-2] = pColumn;
4729 pConst->apExpr[pConst->nConst*2-1] = pValue;
4730 }
4731}
4732
4733/*
4734** Find all terms of COLUMN=VALUE or VALUE=COLUMN in pExpr where VALUE
4735** is a constant expression and where the term must be true because it
4736** is part of the AND-connected terms of the expression. For each term
4737** found, add it to the pConst structure.
4738*/
4739static void findConstInWhere(WhereConst *pConst, Expr *pExpr){
4740 Expr *pRight, *pLeft;
4741 if( NEVER(pExpr==0) ) return;
4742 if( ExprHasProperty(pExpr, pConst->mExcludeOn) ){
4743 testcase( ExprHasProperty(pExpr, EP_OuterON) );
4744 testcase( ExprHasProperty(pExpr, EP_InnerON) );
4745 return;
4746 }
4747 if( pExpr->op==TK_AND ){
4748 findConstInWhere(pConst, pExpr->pRight);
4749 findConstInWhere(pConst, pExpr->pLeft);
4750 return;
4751 }
4752 if( pExpr->op!=TK_EQ ) return;
4753 pRight = pExpr->pRight;
4754 pLeft = pExpr->pLeft;
4755 assert( pRight!=0 );
4756 assert( pLeft!=0 );
4757 if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){
4758 constInsert(pConst,pRight,pLeft,pExpr);
4759 }
4760 if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){
4761 constInsert(pConst,pLeft,pRight,pExpr);
4762 }
4763}
4764
4765/*
4766** This is a helper function for Walker callback propagateConstantExprRewrite().
4767**
4768** Argument pExpr is a candidate expression to be replaced by a value. If
4769** pExpr is equivalent to one of the columns named in pWalker->u.pConst,
4770** then overwrite it with the corresponding value. Except, do not do so
4771** if argument bIgnoreAffBlob is non-zero and the affinity of pExpr
4772** is SQLITE_AFF_BLOB.
4773*/
4774static int propagateConstantExprRewriteOne(
4775 WhereConst *pConst,
4776 Expr *pExpr,
4777 int bIgnoreAffBlob
4778){
4779 int i;
4780 if( pConst->pOomFault[0] ) return WRC_Prune;
4781 if( pExpr->op!=TK_COLUMN ) return WRC_Continue;
4782 if( ExprHasProperty(pExpr, EP_FixedCol|pConst->mExcludeOn) ){
4783 testcase( ExprHasProperty(pExpr, EP_FixedCol) );
4784 testcase( ExprHasProperty(pExpr, EP_OuterON) );
4785 testcase( ExprHasProperty(pExpr, EP_InnerON) );
4786 return WRC_Continue;
4787 }
4788 for(i=0; i<pConst->nConst; i++){
4789 Expr *pColumn = pConst->apExpr[i*2];
4790 if( pColumn==pExpr ) continue;
4791 if( pColumn->iTable!=pExpr->iTable ) continue;
4792 if( pColumn->iColumn!=pExpr->iColumn ) continue;
4793 if( bIgnoreAffBlob && sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){
4794 break;
4795 }
4796 /* A match is found. Add the EP_FixedCol property */
4797 pConst->nChng++;
4798 ExprClearProperty(pExpr, EP_Leaf);
4799 ExprSetProperty(pExpr, EP_FixedCol);
4800 assert( pExpr->pLeft==0 );
4801 pExpr->pLeft = sqlite3ExprDup(pConst->pParse->db, pConst->apExpr[i*2+1], 0);
4802 if( pConst->pParse->db->mallocFailed ) return WRC_Prune;
4803 break;
4804 }
4805 return WRC_Prune;
4806}
4807
4808/*
4809** This is a Walker expression callback. pExpr is a node from the WHERE
4810** clause of a SELECT statement. This function examines pExpr to see if
4811** any substitutions based on the contents of pWalker->u.pConst should
4812** be made to pExpr or its immediate children.
4813**
4814** A substitution is made if:
4815**
4816** + pExpr is a column with an affinity other than BLOB that matches
4817** one of the columns in pWalker->u.pConst, or
4818**
4819** + pExpr is a binary comparison operator (=, <=, >=, <, >) that
4820** uses an affinity other than TEXT and one of its immediate
4821** children is a column that matches one of the columns in
4822** pWalker->u.pConst.
4823*/
4824static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){
4825 WhereConst *pConst = pWalker->u.pConst;
4826 assert( TK_GT==TK_EQ+1 );
4827 assert( TK_LE==TK_EQ+2 );
4828 assert( TK_LT==TK_EQ+3 );
4829 assert( TK_GE==TK_EQ+4 );
4830 if( pConst->bHasAffBlob ){
4831 if( (pExpr->op>=TK_EQ && pExpr->op<=TK_GE)
4832 || pExpr->op==TK_IS
4833 ){
4834 propagateConstantExprRewriteOne(pConst, pExpr->pLeft, 0);
4835 if( pConst->pOomFault[0] ) return WRC_Prune;
4836 if( sqlite3ExprAffinity(pExpr->pLeft)!=SQLITE_AFF_TEXT ){
4837 propagateConstantExprRewriteOne(pConst, pExpr->pRight, 0);
4838 }
4839 }
4840 }
4841 return propagateConstantExprRewriteOne(pConst, pExpr, pConst->bHasAffBlob);
4842}
4843
4844/*
4845** The WHERE-clause constant propagation optimization.
4846**
4847** If the WHERE clause contains terms of the form COLUMN=CONSTANT or
4848** CONSTANT=COLUMN that are top-level AND-connected terms that are not
4849** part of a ON clause from a LEFT JOIN, then throughout the query
4850** replace all other occurrences of COLUMN with CONSTANT.
4851**
4852** For example, the query:
4853**
4854** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=t1.a AND t3.c=t2.b
4855**
4856** Is transformed into
4857**
4858** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=39 AND t3.c=39
4859**
4860** Return true if any transformations where made and false if not.
4861**
4862** Implementation note: Constant propagation is tricky due to affinity
4863** and collating sequence interactions. Consider this example:
4864**
4865** CREATE TABLE t1(a INT,b TEXT);
4866** INSERT INTO t1 VALUES(123,'0123');
4867** SELECT * FROM t1 WHERE a=123 AND b=a;
4868** SELECT * FROM t1 WHERE a=123 AND b=123;
4869**
4870** The two SELECT statements above should return different answers. b=a
4871** is alway true because the comparison uses numeric affinity, but b=123
4872** is false because it uses text affinity and '0123' is not the same as '123'.
4873** To work around this, the expression tree is not actually changed from
4874** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol
4875** and the "123" value is hung off of the pLeft pointer. Code generator
4876** routines know to generate the constant "123" instead of looking up the
4877** column value. Also, to avoid collation problems, this optimization is
4878** only attempted if the "a=123" term uses the default BINARY collation.
4879**
4880** 2021-05-25 forum post 6a06202608: Another troublesome case is...
4881**
4882** CREATE TABLE t1(x);
4883** INSERT INTO t1 VALUES(10.0);
4884** SELECT 1 FROM t1 WHERE x=10 AND x LIKE 10;
4885**
4886** The query should return no rows, because the t1.x value is '10.0' not '10'
4887** and '10.0' is not LIKE '10'. But if we are not careful, the first WHERE
4888** term "x=10" will cause the second WHERE term to become "10 LIKE 10",
4889** resulting in a false positive. To avoid this, constant propagation for
4890** columns with BLOB affinity is only allowed if the constant is used with
4891** operators ==, <=, <, >=, >, or IS in a way that will cause the correct
4892** type conversions to occur. See logic associated with the bHasAffBlob flag
4893** for details.
4894*/
4895static int propagateConstants(
4896 Parse *pParse, /* The parsing context */
4897 Select *p /* The query in which to propagate constants */
4898){
4899 WhereConst x;
4900 Walker w;
4901 int nChng = 0;
4902 x.pParse = pParse;
4903 x.pOomFault = &pParse->db->mallocFailed;
4904 do{
4905 x.nConst = 0;
4906 x.nChng = 0;
4907 x.apExpr = 0;
4908 x.bHasAffBlob = 0;
4909 if( ALWAYS(p->pSrc!=0)
4910 && p->pSrc->nSrc>0
4911 && (p->pSrc->a[0].fg.jointype & JT_LTORJ)!=0
4912 ){
4913 /* Do not propagate constants on any ON clause if there is a
4914 ** RIGHT JOIN anywhere in the query */
4915 x.mExcludeOn = EP_InnerON | EP_OuterON;
4916 }else{
4917 /* Do not propagate constants through the ON clause of a LEFT JOIN */
4918 x.mExcludeOn = EP_OuterON;
4919 }
4920 findConstInWhere(&x, p->pWhere);
4921 if( x.nConst ){
4922 memset(&w, 0, sizeof(w));
4923 w.pParse = pParse;
4924 w.xExprCallback = propagateConstantExprRewrite;
4925 w.xSelectCallback = sqlite3SelectWalkNoop;
4926 w.xSelectCallback2 = 0;
4927 w.walkerDepth = 0;
4928 w.u.pConst = &x;
4929 sqlite3WalkExpr(&w, p->pWhere);
4930 sqlite3DbFree(x.pParse->db, x.apExpr);
4931 nChng += x.nChng;
4932 }
4933 }while( x.nChng );
4934 return nChng;
4935}
4936
4937#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
4938# if !defined(SQLITE_OMIT_WINDOWFUNC)
4939/*
4940** This function is called to determine whether or not it is safe to
4941** push WHERE clause expression pExpr down to FROM clause sub-query
4942** pSubq, which contains at least one window function. Return 1
4943** if it is safe and the expression should be pushed down, or 0
4944** otherwise.
4945**
4946** It is only safe to push the expression down if it consists only
4947** of constants and copies of expressions that appear in the PARTITION
4948** BY clause of all window function used by the sub-query. It is safe
4949** to filter out entire partitions, but not rows within partitions, as
4950** this may change the results of the window functions.
4951**
4952** At the time this function is called it is guaranteed that
4953**
4954** * the sub-query uses only one distinct window frame, and
4955** * that the window frame has a PARTITION BY clase.
4956*/
4957static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){
4958 assert( pSubq->pWin->pPartition );
4959 assert( (pSubq->selFlags & SF_MultiPart)==0 );
4960 assert( pSubq->pPrior==0 );
4961 return sqlite3ExprIsConstantOrGroupBy(pParse, pExpr, pSubq->pWin->pPartition);
4962}
4963# endif /* SQLITE_OMIT_WINDOWFUNC */
4964#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
4965
4966#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
4967/*
4968** Make copies of relevant WHERE clause terms of the outer query into
4969** the WHERE clause of subquery. Example:
4970**
4971** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1) WHERE x=5 AND y=10;
4972**
4973** Transformed into:
4974**
4975** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1 WHERE a=5 AND c-d=10)
4976** WHERE x=5 AND y=10;
4977**
4978** The hope is that the terms added to the inner query will make it more
4979** efficient.
4980**
4981** Do not attempt this optimization if:
4982**
4983** (1) (** This restriction was removed on 2017-09-29. We used to
4984** disallow this optimization for aggregate subqueries, but now
4985** it is allowed by putting the extra terms on the HAVING clause.
4986** The added HAVING clause is pointless if the subquery lacks
4987** a GROUP BY clause. But such a HAVING clause is also harmless
4988** so there does not appear to be any reason to add extra logic
4989** to suppress it. **)
4990**
4991** (2) The inner query is the recursive part of a common table expression.
4992**
4993** (3) The inner query has a LIMIT clause (since the changes to the WHERE
4994** clause would change the meaning of the LIMIT).
4995**
4996** (4) The inner query is the right operand of a LEFT JOIN and the
4997** expression to be pushed down does not come from the ON clause
4998** on that LEFT JOIN.
4999**
5000** (5) The WHERE clause expression originates in the ON or USING clause
5001** of a LEFT JOIN where iCursor is not the right-hand table of that
5002** left join. An example:
5003**
5004** SELECT *
5005** FROM (SELECT 1 AS a1 UNION ALL SELECT 2) AS aa
5006** JOIN (SELECT 1 AS b2 UNION ALL SELECT 2) AS bb ON (a1=b2)
5007** LEFT JOIN (SELECT 8 AS c3 UNION ALL SELECT 9) AS cc ON (b2=2);
5008**
5009** The correct answer is three rows: (1,1,NULL),(2,2,8),(2,2,9).
5010** But if the (b2=2) term were to be pushed down into the bb subquery,
5011** then the (1,1,NULL) row would be suppressed.
5012**
5013** (6) Window functions make things tricky as changes to the WHERE clause
5014** of the inner query could change the window over which window
5015** functions are calculated. Therefore, do not attempt the optimization
5016** if:
5017**
5018** (6a) The inner query uses multiple incompatible window partitions.
5019**
5020** (6b) The inner query is a compound and uses window-functions.
5021**
5022** (6c) The WHERE clause does not consist entirely of constants and
5023** copies of expressions found in the PARTITION BY clause of
5024** all window-functions used by the sub-query. It is safe to
5025** filter out entire partitions, as this does not change the
5026** window over which any window-function is calculated.
5027**
5028** (7) The inner query is a Common Table Expression (CTE) that should
5029** be materialized. (This restriction is implemented in the calling
5030** routine.)
5031**
5032** (8) The subquery may not be a compound that uses UNION, INTERSECT,
5033** or EXCEPT. (We could, perhaps, relax this restriction to allow
5034** this case if none of the comparisons operators between left and
5035** right arms of the compound use a collation other than BINARY.
5036** But it is a lot of work to check that case for an obscure and
5037** minor optimization, so we omit it for now.)
5038**
5039** Return 0 if no changes are made and non-zero if one or more WHERE clause
5040** terms are duplicated into the subquery.
5041*/
5042static int pushDownWhereTerms(
5043 Parse *pParse, /* Parse context (for malloc() and error reporting) */
5044 Select *pSubq, /* The subquery whose WHERE clause is to be augmented */
5045 Expr *pWhere, /* The WHERE clause of the outer query */
5046 SrcItem *pSrc /* The subquery term of the outer FROM clause */
5047){
5048 Expr *pNew;
5049 int nChng = 0;
5050 if( pWhere==0 ) return 0;
5051 if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0;
5052 if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0;
5053
5054#ifndef SQLITE_OMIT_WINDOWFUNC
5055 if( pSubq->pPrior ){
5056 Select *pSel;
5057 for(pSel=pSubq; pSel; pSel=pSel->pPrior){
5058 u8 op = pSel->op;
5059 assert( op==TK_ALL || op==TK_SELECT
5060 || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT );
5061 if( op!=TK_ALL && op!=TK_SELECT ) return 0; /* restriction (8) */
5062 if( pSel->pWin ) return 0; /* restriction (6b) */
5063 }
5064 }else{
5065 if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0;
5066 }
5067#endif
5068
5069#ifdef SQLITE_DEBUG
5070 /* Only the first term of a compound can have a WITH clause. But make
5071 ** sure no other terms are marked SF_Recursive in case something changes
5072 ** in the future.
5073 */
5074 {
5075 Select *pX;
5076 for(pX=pSubq; pX; pX=pX->pPrior){
5077 assert( (pX->selFlags & (SF_Recursive))==0 );
5078 }
5079 }
5080#endif
5081
5082 if( pSubq->pLimit!=0 ){
5083 return 0; /* restriction (3) */
5084 }
5085 while( pWhere->op==TK_AND ){
5086 nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrc);
5087 pWhere = pWhere->pLeft;
5088 }
5089
5090#if 0 /* Legacy code. Checks now done by sqlite3ExprIsTableConstraint() */
5091 if( isLeftJoin
5092 && (ExprHasProperty(pWhere,EP_OuterON)==0
5093 || pWhere->w.iJoin!=iCursor)
5094 ){
5095 return 0; /* restriction (4) */
5096 }
5097 if( ExprHasProperty(pWhere,EP_OuterON)
5098 && pWhere->w.iJoin!=iCursor
5099 ){
5100 return 0; /* restriction (5) */
5101 }
5102#endif
5103
5104 if( sqlite3ExprIsTableConstraint(pWhere, pSrc) ){
5105 nChng++;
5106 pSubq->selFlags |= SF_PushDown;
5107 while( pSubq ){
5108 SubstContext x;
5109 pNew = sqlite3ExprDup(pParse->db, pWhere, 0);
5110 unsetJoinExpr(pNew, -1, 1);
5111 x.pParse = pParse;
5112 x.iTable = pSrc->iCursor;
5113 x.iNewTable = pSrc->iCursor;
5114 x.isOuterJoin = 0;
5115 x.pEList = pSubq->pEList;
5116 x.pCList = findLeftmostExprlist(pSubq);
5117 pNew = substExpr(&x, pNew);
5118#ifndef SQLITE_OMIT_WINDOWFUNC
5119 if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){
5120 /* Restriction 6c has prevented push-down in this case */
5121 sqlite3ExprDelete(pParse->db, pNew);
5122 nChng--;
5123 break;
5124 }
5125#endif
5126 if( pSubq->selFlags & SF_Aggregate ){
5127 pSubq->pHaving = sqlite3ExprAnd(pParse, pSubq->pHaving, pNew);
5128 }else{
5129 pSubq->pWhere = sqlite3ExprAnd(pParse, pSubq->pWhere, pNew);
5130 }
5131 pSubq = pSubq->pPrior;
5132 }
5133 }
5134 return nChng;
5135}
5136#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
5137
5138/*
5139** The pFunc is the only aggregate function in the query. Check to see
5140** if the query is a candidate for the min/max optimization.
5141**
5142** If the query is a candidate for the min/max optimization, then set
5143** *ppMinMax to be an ORDER BY clause to be used for the optimization
5144** and return either WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX depending on
5145** whether pFunc is a min() or max() function.
5146**
5147** If the query is not a candidate for the min/max optimization, return
5148** WHERE_ORDERBY_NORMAL (which must be zero).
5149**
5150** This routine must be called after aggregate functions have been
5151** located but before their arguments have been subjected to aggregate
5152** analysis.
5153*/
5154static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){
5155 int eRet = WHERE_ORDERBY_NORMAL; /* Return value */
5156 ExprList *pEList; /* Arguments to agg function */
5157 const char *zFunc; /* Name of aggregate function pFunc */
5158 ExprList *pOrderBy;
5159 u8 sortFlags = 0;
5160
5161 assert( *ppMinMax==0 );
5162 assert( pFunc->op==TK_AGG_FUNCTION );
5163 assert( !IsWindowFunc(pFunc) );
5164 assert( ExprUseXList(pFunc) );
5165 pEList = pFunc->x.pList;
5166 if( pEList==0
5167 || pEList->nExpr!=1
5168 || ExprHasProperty(pFunc, EP_WinFunc)
5169 || OptimizationDisabled(db, SQLITE_MinMaxOpt)
5170 ){
5171 return eRet;
5172 }
5173 assert( !ExprHasProperty(pFunc, EP_IntValue) );
5174 zFunc = pFunc->u.zToken;
5175 if( sqlite3StrICmp(zFunc, "min")==0 ){
5176 eRet = WHERE_ORDERBY_MIN;
5177 if( sqlite3ExprCanBeNull(pEList->a[0].pExpr) ){
5178 sortFlags = KEYINFO_ORDER_BIGNULL;
5179 }
5180 }else if( sqlite3StrICmp(zFunc, "max")==0 ){
5181 eRet = WHERE_ORDERBY_MAX;
5182 sortFlags = KEYINFO_ORDER_DESC;
5183 }else{
5184 return eRet;
5185 }
5186 *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0);
5187 assert( pOrderBy!=0 || db->mallocFailed );
5188 if( pOrderBy ) pOrderBy->a[0].fg.sortFlags = sortFlags;
5189 return eRet;
5190}
5191
5192/*
5193** The select statement passed as the first argument is an aggregate query.
5194** The second argument is the associated aggregate-info object. This
5195** function tests if the SELECT is of the form:
5196**
5197** SELECT count(*) FROM <tbl>
5198**
5199** where table is a database table, not a sub-select or view. If the query
5200** does match this pattern, then a pointer to the Table object representing
5201** <tbl> is returned. Otherwise, NULL is returned.
5202**
5203** This routine checks to see if it is safe to use the count optimization.
5204** A correct answer is still obtained (though perhaps more slowly) if
5205** this routine returns NULL when it could have returned a table pointer.
5206** But returning the pointer when NULL should have been returned can
5207** result in incorrect answers and/or crashes. So, when in doubt, return NULL.
5208*/
5209static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){
5210 Table *pTab;
5211 Expr *pExpr;
5212
5213 assert( !p->pGroupBy );
5214
5215 if( p->pWhere
5216 || p->pEList->nExpr!=1
5217 || p->pSrc->nSrc!=1
5218 || p->pSrc->a[0].pSelect
5219 || pAggInfo->nFunc!=1
5220 || p->pHaving
5221 ){
5222 return 0;
5223 }
5224 pTab = p->pSrc->a[0].pTab;
5225 assert( pTab!=0 );
5226 assert( !IsView(pTab) );
5227 if( !IsOrdinaryTable(pTab) ) return 0;
5228 pExpr = p->pEList->a[0].pExpr;
5229 assert( pExpr!=0 );
5230 if( pExpr->op!=TK_AGG_FUNCTION ) return 0;
5231 if( pExpr->pAggInfo!=pAggInfo ) return 0;
5232 if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0;
5233 assert( pAggInfo->aFunc[0].pFExpr==pExpr );
5234 testcase( ExprHasProperty(pExpr, EP_Distinct) );
5235 testcase( ExprHasProperty(pExpr, EP_WinFunc) );
5236 if( ExprHasProperty(pExpr, EP_Distinct|EP_WinFunc) ) return 0;
5237
5238 return pTab;
5239}
5240
5241/*
5242** If the source-list item passed as an argument was augmented with an
5243** INDEXED BY clause, then try to locate the specified index. If there
5244** was such a clause and the named index cannot be found, return
5245** SQLITE_ERROR and leave an error in pParse. Otherwise, populate
5246** pFrom->pIndex and return SQLITE_OK.
5247*/
5248int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){
5249 Table *pTab = pFrom->pTab;
5250 char *zIndexedBy = pFrom->u1.zIndexedBy;
5251 Index *pIdx;
5252 assert( pTab!=0 );
5253 assert( pFrom->fg.isIndexedBy!=0 );
5254
5255 for(pIdx=pTab->pIndex;
5256 pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy);
5257 pIdx=pIdx->pNext
5258 );
5259 if( !pIdx ){
5260 sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0);
5261 pParse->checkSchema = 1;
5262 return SQLITE_ERROR;
5263 }
5264 assert( pFrom->fg.isCte==0 );
5265 pFrom->u2.pIBIndex = pIdx;
5266 return SQLITE_OK;
5267}
5268
5269/*
5270** Detect compound SELECT statements that use an ORDER BY clause with
5271** an alternative collating sequence.
5272**
5273** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ...
5274**
5275** These are rewritten as a subquery:
5276**
5277** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2)
5278** ORDER BY ... COLLATE ...
5279**
5280** This transformation is necessary because the multiSelectOrderBy() routine
5281** above that generates the code for a compound SELECT with an ORDER BY clause
5282** uses a merge algorithm that requires the same collating sequence on the
5283** result columns as on the ORDER BY clause. See ticket
5284** http://www.sqlite.org/src/info/6709574d2a
5285**
5286** This transformation is only needed for EXCEPT, INTERSECT, and UNION.
5287** The UNION ALL operator works fine with multiSelectOrderBy() even when
5288** there are COLLATE terms in the ORDER BY.
5289*/
5290static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
5291 int i;
5292 Select *pNew;
5293 Select *pX;
5294 sqlite3 *db;
5295 struct ExprList_item *a;
5296 SrcList *pNewSrc;
5297 Parse *pParse;
5298 Token dummy;
5299
5300 if( p->pPrior==0 ) return WRC_Continue;
5301 if( p->pOrderBy==0 ) return WRC_Continue;
5302 for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){}
5303 if( pX==0 ) return WRC_Continue;
5304 a = p->pOrderBy->a;
5305#ifndef SQLITE_OMIT_WINDOWFUNC
5306 /* If iOrderByCol is already non-zero, then it has already been matched
5307 ** to a result column of the SELECT statement. This occurs when the
5308 ** SELECT is rewritten for window-functions processing and then passed
5309 ** to sqlite3SelectPrep() and similar a second time. The rewriting done
5310 ** by this function is not required in this case. */
5311 if( a[0].u.x.iOrderByCol ) return WRC_Continue;
5312#endif
5313 for(i=p->pOrderBy->nExpr-1; i>=0; i--){
5314 if( a[i].pExpr->flags & EP_Collate ) break;
5315 }
5316 if( i<0 ) return WRC_Continue;
5317
5318 /* If we reach this point, that means the transformation is required. */
5319
5320 pParse = pWalker->pParse;
5321 db = pParse->db;
5322 pNew = sqlite3DbMallocZero(db, sizeof(*pNew) );
5323 if( pNew==0 ) return WRC_Abort;
5324 memset(&dummy, 0, sizeof(dummy));
5325 pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0);
5326 if( pNewSrc==0 ) return WRC_Abort;
5327 *pNew = *p;
5328 p->pSrc = pNewSrc;
5329 p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0));
5330 p->op = TK_SELECT;
5331 p->pWhere = 0;
5332 pNew->pGroupBy = 0;
5333 pNew->pHaving = 0;
5334 pNew->pOrderBy = 0;
5335 p->pPrior = 0;
5336 p->pNext = 0;
5337 p->pWith = 0;
5338#ifndef SQLITE_OMIT_WINDOWFUNC
5339 p->pWinDefn = 0;
5340#endif
5341 p->selFlags &= ~SF_Compound;
5342 assert( (p->selFlags & SF_Converted)==0 );
5343 p->selFlags |= SF_Converted;
5344 assert( pNew->pPrior!=0 );
5345 pNew->pPrior->pNext = pNew;
5346 pNew->pLimit = 0;
5347 return WRC_Continue;
5348}
5349
5350/*
5351** Check to see if the FROM clause term pFrom has table-valued function
5352** arguments. If it does, leave an error message in pParse and return
5353** non-zero, since pFrom is not allowed to be a table-valued function.
5354*/
5355static int cannotBeFunction(Parse *pParse, SrcItem *pFrom){
5356 if( pFrom->fg.isTabFunc ){
5357 sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName);
5358 return 1;
5359 }
5360 return 0;
5361}
5362
5363#ifndef SQLITE_OMIT_CTE
5364/*
5365** Argument pWith (which may be NULL) points to a linked list of nested
5366** WITH contexts, from inner to outermost. If the table identified by
5367** FROM clause element pItem is really a common-table-expression (CTE)
5368** then return a pointer to the CTE definition for that table. Otherwise
5369** return NULL.
5370**
5371** If a non-NULL value is returned, set *ppContext to point to the With
5372** object that the returned CTE belongs to.
5373*/
5374static struct Cte *searchWith(
5375 With *pWith, /* Current innermost WITH clause */
5376 SrcItem *pItem, /* FROM clause element to resolve */
5377 With **ppContext /* OUT: WITH clause return value belongs to */
5378){
5379 const char *zName = pItem->zName;
5380 With *p;
5381 assert( pItem->zDatabase==0 );
5382 assert( zName!=0 );
5383 for(p=pWith; p; p=p->pOuter){
5384 int i;
5385 for(i=0; i<p->nCte; i++){
5386 if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){
5387 *ppContext = p;
5388 return &p->a[i];
5389 }
5390 }
5391 if( p->bView ) break;
5392 }
5393 return 0;
5394}
5395
5396/* The code generator maintains a stack of active WITH clauses
5397** with the inner-most WITH clause being at the top of the stack.
5398**
5399** This routine pushes the WITH clause passed as the second argument
5400** onto the top of the stack. If argument bFree is true, then this
5401** WITH clause will never be popped from the stack but should instead
5402** be freed along with the Parse object. In other cases, when
5403** bFree==0, the With object will be freed along with the SELECT
5404** statement with which it is associated.
5405**
5406** This routine returns a copy of pWith. Or, if bFree is true and
5407** the pWith object is destroyed immediately due to an OOM condition,
5408** then this routine return NULL.
5409**
5410** If bFree is true, do not continue to use the pWith pointer after
5411** calling this routine, Instead, use only the return value.
5412*/
5413With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
5414 if( pWith ){
5415 if( bFree ){
5416 pWith = (With*)sqlite3ParserAddCleanup(pParse,
5417 (void(*)(sqlite3*,void*))sqlite3WithDelete,
5418 pWith);
5419 if( pWith==0 ) return 0;
5420 }
5421 if( pParse->nErr==0 ){
5422 assert( pParse->pWith!=pWith );
5423 pWith->pOuter = pParse->pWith;
5424 pParse->pWith = pWith;
5425 }
5426 }
5427 return pWith;
5428}
5429
5430/*
5431** This function checks if argument pFrom refers to a CTE declared by
5432** a WITH clause on the stack currently maintained by the parser (on the
5433** pParse->pWith linked list). And if currently processing a CTE
5434** CTE expression, through routine checks to see if the reference is
5435** a recursive reference to the CTE.
5436**
5437** If pFrom matches a CTE according to either of these two above, pFrom->pTab
5438** and other fields are populated accordingly.
5439**
5440** Return 0 if no match is found.
5441** Return 1 if a match is found.
5442** Return 2 if an error condition is detected.
5443*/
5444static int resolveFromTermToCte(
5445 Parse *pParse, /* The parsing context */
5446 Walker *pWalker, /* Current tree walker */
5447 SrcItem *pFrom /* The FROM clause term to check */
5448){
5449 Cte *pCte; /* Matched CTE (or NULL if no match) */
5450 With *pWith; /* The matching WITH */
5451
5452 assert( pFrom->pTab==0 );
5453 if( pParse->pWith==0 ){
5454 /* There are no WITH clauses in the stack. No match is possible */
5455 return 0;
5456 }
5457 if( pParse->nErr ){
5458 /* Prior errors might have left pParse->pWith in a goofy state, so
5459 ** go no further. */
5460 return 0;
5461 }
5462 if( pFrom->zDatabase!=0 ){
5463 /* The FROM term contains a schema qualifier (ex: main.t1) and so
5464 ** it cannot possibly be a CTE reference. */
5465 return 0;
5466 }
5467 if( pFrom->fg.notCte ){
5468 /* The FROM term is specifically excluded from matching a CTE.
5469 ** (1) It is part of a trigger that used to have zDatabase but had
5470 ** zDatabase removed by sqlite3FixTriggerStep().
5471 ** (2) This is the first term in the FROM clause of an UPDATE.
5472 */
5473 return 0;
5474 }
5475 pCte = searchWith(pParse->pWith, pFrom, &pWith);
5476 if( pCte ){
5477 sqlite3 *db = pParse->db;
5478 Table *pTab;
5479 ExprList *pEList;
5480 Select *pSel;
5481 Select *pLeft; /* Left-most SELECT statement */
5482 Select *pRecTerm; /* Left-most recursive term */
5483 int bMayRecursive; /* True if compound joined by UNION [ALL] */
5484 With *pSavedWith; /* Initial value of pParse->pWith */
5485 int iRecTab = -1; /* Cursor for recursive table */
5486 CteUse *pCteUse;
5487
5488 /* If pCte->zCteErr is non-NULL at this point, then this is an illegal
5489 ** recursive reference to CTE pCte. Leave an error in pParse and return
5490 ** early. If pCte->zCteErr is NULL, then this is not a recursive reference.
5491 ** In this case, proceed. */
5492 if( pCte->zCteErr ){
5493 sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName);
5494 return 2;
5495 }
5496 if( cannotBeFunction(pParse, pFrom) ) return 2;
5497
5498 assert( pFrom->pTab==0 );
5499 pTab = sqlite3DbMallocZero(db, sizeof(Table));
5500 if( pTab==0 ) return 2;
5501 pCteUse = pCte->pUse;
5502 if( pCteUse==0 ){
5503 pCte->pUse = pCteUse = sqlite3DbMallocZero(db, sizeof(pCteUse[0]));
5504 if( pCteUse==0
5505 || sqlite3ParserAddCleanup(pParse,sqlite3DbFree,pCteUse)==0
5506 ){
5507 sqlite3DbFree(db, pTab);
5508 return 2;
5509 }
5510 pCteUse->eM10d = pCte->eM10d;
5511 }
5512 pFrom->pTab = pTab;
5513 pTab->nTabRef = 1;
5514 pTab->zName = sqlite3DbStrDup(db, pCte->zName);
5515 pTab->iPKey = -1;
5516 pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );
5517 pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid;
5518 pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0);
5519 if( db->mallocFailed ) return 2;
5520 pFrom->pSelect->selFlags |= SF_CopyCte;
5521 assert( pFrom->pSelect );
5522 if( pFrom->fg.isIndexedBy ){
5523 sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy);
5524 return 2;
5525 }
5526 pFrom->fg.isCte = 1;
5527 pFrom->u2.pCteUse = pCteUse;
5528 pCteUse->nUse++;
5529 if( pCteUse->nUse>=2 && pCteUse->eM10d==M10d_Any ){
5530 pCteUse->eM10d = M10d_Yes;
5531 }
5532
5533 /* Check if this is a recursive CTE. */
5534 pRecTerm = pSel = pFrom->pSelect;
5535 bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION );
5536 while( bMayRecursive && pRecTerm->op==pSel->op ){
5537 int i;
5538 SrcList *pSrc = pRecTerm->pSrc;
5539 assert( pRecTerm->pPrior!=0 );
5540 for(i=0; i<pSrc->nSrc; i++){
5541 SrcItem *pItem = &pSrc->a[i];
5542 if( pItem->zDatabase==0
5543 && pItem->zName!=0
5544 && 0==sqlite3StrICmp(pItem->zName, pCte->zName)
5545 ){
5546 pItem->pTab = pTab;
5547 pTab->nTabRef++;
5548 pItem->fg.isRecursive = 1;
5549 if( pRecTerm->selFlags & SF_Recursive ){
5550 sqlite3ErrorMsg(pParse,
5551 "multiple references to recursive table: %s", pCte->zName
5552 );
5553 return 2;
5554 }
5555 pRecTerm->selFlags |= SF_Recursive;
5556 if( iRecTab<0 ) iRecTab = pParse->nTab++;
5557 pItem->iCursor = iRecTab;
5558 }
5559 }
5560 if( (pRecTerm->selFlags & SF_Recursive)==0 ) break;
5561 pRecTerm = pRecTerm->pPrior;
5562 }
5563
5564 pCte->zCteErr = "circular reference: %s";
5565 pSavedWith = pParse->pWith;
5566 pParse->pWith = pWith;
5567 if( pSel->selFlags & SF_Recursive ){
5568 int rc;
5569 assert( pRecTerm!=0 );
5570 assert( (pRecTerm->selFlags & SF_Recursive)==0 );
5571 assert( pRecTerm->pNext!=0 );
5572 assert( (pRecTerm->pNext->selFlags & SF_Recursive)!=0 );
5573 assert( pRecTerm->pWith==0 );
5574 pRecTerm->pWith = pSel->pWith;
5575 rc = sqlite3WalkSelect(pWalker, pRecTerm);
5576 pRecTerm->pWith = 0;
5577 if( rc ){
5578 pParse->pWith = pSavedWith;
5579 return 2;
5580 }
5581 }else{
5582 if( sqlite3WalkSelect(pWalker, pSel) ){
5583 pParse->pWith = pSavedWith;
5584 return 2;
5585 }
5586 }
5587 pParse->pWith = pWith;
5588
5589 for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior);
5590 pEList = pLeft->pEList;
5591 if( pCte->pCols ){
5592 if( pEList && pEList->nExpr!=pCte->pCols->nExpr ){
5593 sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns",
5594 pCte->zName, pEList->nExpr, pCte->pCols->nExpr
5595 );
5596 pParse->pWith = pSavedWith;
5597 return 2;
5598 }
5599 pEList = pCte->pCols;
5600 }
5601
5602 sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol);
5603 if( bMayRecursive ){
5604 if( pSel->selFlags & SF_Recursive ){
5605 pCte->zCteErr = "multiple recursive references: %s";
5606 }else{
5607 pCte->zCteErr = "recursive reference in a subquery: %s";
5608 }
5609 sqlite3WalkSelect(pWalker, pSel);
5610 }
5611 pCte->zCteErr = 0;
5612 pParse->pWith = pSavedWith;
5613 return 1; /* Success */
5614 }
5615 return 0; /* No match */
5616}
5617#endif
5618
5619#ifndef SQLITE_OMIT_CTE
5620/*
5621** If the SELECT passed as the second argument has an associated WITH
5622** clause, pop it from the stack stored as part of the Parse object.
5623**
5624** This function is used as the xSelectCallback2() callback by
5625** sqlite3SelectExpand() when walking a SELECT tree to resolve table
5626** names and other FROM clause elements.
5627*/
5628void sqlite3SelectPopWith(Walker *pWalker, Select *p){
5629 Parse *pParse = pWalker->pParse;
5630 if( OK_IF_ALWAYS_TRUE(pParse->pWith) && p->pPrior==0 ){
5631 With *pWith = findRightmost(p)->pWith;
5632 if( pWith!=0 ){
5633 assert( pParse->pWith==pWith || pParse->nErr );
5634 pParse->pWith = pWith->pOuter;
5635 }
5636 }
5637}
5638#endif
5639
5640/*
5641** The SrcItem structure passed as the second argument represents a
5642** sub-query in the FROM clause of a SELECT statement. This function
5643** allocates and populates the SrcItem.pTab object. If successful,
5644** SQLITE_OK is returned. Otherwise, if an OOM error is encountered,
5645** SQLITE_NOMEM.
5646*/
5647int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){
5648 Select *pSel = pFrom->pSelect;
5649 Table *pTab;
5650
5651 assert( pSel );
5652 pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table));
5653 if( pTab==0 ) return SQLITE_NOMEM;
5654 pTab->nTabRef = 1;
5655 if( pFrom->zAlias ){
5656 pTab->zName = sqlite3DbStrDup(pParse->db, pFrom->zAlias);
5657 }else{
5658 pTab->zName = sqlite3MPrintf(pParse->db, "%!S", pFrom);
5659 }
5660 while( pSel->pPrior ){ pSel = pSel->pPrior; }
5661 sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol);
5662 pTab->iPKey = -1;
5663 pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );
5664#ifndef SQLITE_ALLOW_ROWID_IN_VIEW
5665 /* The usual case - do not allow ROWID on a subquery */
5666 pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid;
5667#else
5668 pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */
5669#endif
5670 return pParse->nErr ? SQLITE_ERROR : SQLITE_OK;
5671}
5672
5673
5674/*
5675** Check the N SrcItem objects to the right of pBase. (N might be zero!)
5676** If any of those SrcItem objects have a USING clause containing zName
5677** then return true.
5678**
5679** If N is zero, or none of the N SrcItem objects to the right of pBase
5680** contains a USING clause, or if none of the USING clauses contain zName,
5681** then return false.
5682*/
5683static int inAnyUsingClause(
5684 const char *zName, /* Name we are looking for */
5685 SrcItem *pBase, /* The base SrcItem. Looking at pBase[1] and following */
5686 int N /* How many SrcItems to check */
5687){
5688 while( N>0 ){
5689 N--;
5690 pBase++;
5691 if( pBase->fg.isUsing==0 ) continue;
5692 if( NEVER(pBase->u3.pUsing==0) ) continue;
5693 if( sqlite3IdListIndex(pBase->u3.pUsing, zName)>=0 ) return 1;
5694 }
5695 return 0;
5696}
5697
5698
5699/*
5700** This routine is a Walker callback for "expanding" a SELECT statement.
5701** "Expanding" means to do the following:
5702**
5703** (1) Make sure VDBE cursor numbers have been assigned to every
5704** element of the FROM clause.
5705**
5706** (2) Fill in the pTabList->a[].pTab fields in the SrcList that
5707** defines FROM clause. When views appear in the FROM clause,
5708** fill pTabList->a[].pSelect with a copy of the SELECT statement
5709** that implements the view. A copy is made of the view's SELECT
5710** statement so that we can freely modify or delete that statement
5711** without worrying about messing up the persistent representation
5712** of the view.
5713**
5714** (3) Add terms to the WHERE clause to accommodate the NATURAL keyword
5715** on joins and the ON and USING clause of joins.
5716**
5717** (4) Scan the list of columns in the result set (pEList) looking
5718** for instances of the "*" operator or the TABLE.* operator.
5719** If found, expand each "*" to be every column in every table
5720** and TABLE.* to be every column in TABLE.
5721**
5722*/
5723static int selectExpander(Walker *pWalker, Select *p){
5724 Parse *pParse = pWalker->pParse;
5725 int i, j, k, rc;
5726 SrcList *pTabList;
5727 ExprList *pEList;
5728 SrcItem *pFrom;
5729 sqlite3 *db = pParse->db;
5730 Expr *pE, *pRight, *pExpr;
5731 u16 selFlags = p->selFlags;
5732 u32 elistFlags = 0;
5733
5734 p->selFlags |= SF_Expanded;
5735 if( db->mallocFailed ){
5736 return WRC_Abort;
5737 }
5738 assert( p->pSrc!=0 );
5739 if( (selFlags & SF_Expanded)!=0 ){
5740 return WRC_Prune;
5741 }
5742 if( pWalker->eCode ){
5743 /* Renumber selId because it has been copied from a view */
5744 p->selId = ++pParse->nSelect;
5745 }
5746 pTabList = p->pSrc;
5747 pEList = p->pEList;
5748 if( pParse->pWith && (p->selFlags & SF_View) ){
5749 if( p->pWith==0 ){
5750 p->pWith = (With*)sqlite3DbMallocZero(db, sizeof(With));
5751 if( p->pWith==0 ){
5752 return WRC_Abort;
5753 }
5754 }
5755 p->pWith->bView = 1;
5756 }
5757 sqlite3WithPush(pParse, p->pWith, 0);
5758
5759 /* Make sure cursor numbers have been assigned to all entries in
5760 ** the FROM clause of the SELECT statement.
5761 */
5762 sqlite3SrcListAssignCursors(pParse, pTabList);
5763
5764 /* Look up every table named in the FROM clause of the select. If
5765 ** an entry of the FROM clause is a subquery instead of a table or view,
5766 ** then create a transient table structure to describe the subquery.
5767 */
5768 for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
5769 Table *pTab;
5770 assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 );
5771 if( pFrom->pTab ) continue;
5772 assert( pFrom->fg.isRecursive==0 );
5773 if( pFrom->zName==0 ){
5774#ifndef SQLITE_OMIT_SUBQUERY
5775 Select *pSel = pFrom->pSelect;
5776 /* A sub-query in the FROM clause of a SELECT */
5777 assert( pSel!=0 );
5778 assert( pFrom->pTab==0 );
5779 if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort;
5780 if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort;
5781#endif
5782#ifndef SQLITE_OMIT_CTE
5783 }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){
5784 if( rc>1 ) return WRC_Abort;
5785 pTab = pFrom->pTab;
5786 assert( pTab!=0 );
5787#endif
5788 }else{
5789 /* An ordinary table or view name in the FROM clause */
5790 assert( pFrom->pTab==0 );
5791 pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom);
5792 if( pTab==0 ) return WRC_Abort;
5793 if( pTab->nTabRef>=0xffff ){
5794 sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535",
5795 pTab->zName);
5796 pFrom->pTab = 0;
5797 return WRC_Abort;
5798 }
5799 pTab->nTabRef++;
5800 if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){
5801 return WRC_Abort;
5802 }
5803#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE)
5804 if( !IsOrdinaryTable(pTab) ){
5805 i16 nCol;
5806 u8 eCodeOrig = pWalker->eCode;
5807 if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort;
5808 assert( pFrom->pSelect==0 );
5809 if( IsView(pTab) ){
5810 if( (db->flags & SQLITE_EnableView)==0
5811 && pTab->pSchema!=db->aDb[1].pSchema
5812 ){
5813 sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited",
5814 pTab->zName);
5815 }
5816 pFrom->pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0);
5817 }
5818#ifndef SQLITE_OMIT_VIRTUALTABLE
5819 else if( ALWAYS(IsVirtual(pTab))
5820 && pFrom->fg.fromDDL
5821 && ALWAYS(pTab->u.vtab.p!=0)
5822 && pTab->u.vtab.p->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0)
5823 ){
5824 sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"",
5825 pTab->zName);
5826 }
5827 assert( SQLITE_VTABRISK_Normal==1 && SQLITE_VTABRISK_High==2 );
5828#endif
5829 nCol = pTab->nCol;
5830 pTab->nCol = -1;
5831 pWalker->eCode = 1; /* Turn on Select.selId renumbering */
5832 sqlite3WalkSelect(pWalker, pFrom->pSelect);
5833 pWalker->eCode = eCodeOrig;
5834 pTab->nCol = nCol;
5835 }
5836#endif
5837 }
5838
5839 /* Locate the index named by the INDEXED BY clause, if any. */
5840 if( pFrom->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pFrom) ){
5841 return WRC_Abort;
5842 }
5843 }
5844
5845 /* Process NATURAL keywords, and ON and USING clauses of joins.
5846 */
5847 assert( db->mallocFailed==0 || pParse->nErr!=0 );
5848 if( pParse->nErr || sqlite3ProcessJoin(pParse, p) ){
5849 return WRC_Abort;
5850 }
5851
5852 /* For every "*" that occurs in the column list, insert the names of
5853 ** all columns in all tables. And for every TABLE.* insert the names
5854 ** of all columns in TABLE. The parser inserted a special expression
5855 ** with the TK_ASTERISK operator for each "*" that it found in the column
5856 ** list. The following code just has to locate the TK_ASTERISK
5857 ** expressions and expand each one to the list of all columns in
5858 ** all tables.
5859 **
5860 ** The first loop just checks to see if there are any "*" operators
5861 ** that need expanding.
5862 */
5863 for(k=0; k<pEList->nExpr; k++){
5864 pE = pEList->a[k].pExpr;
5865 if( pE->op==TK_ASTERISK ) break;
5866 assert( pE->op!=TK_DOT || pE->pRight!=0 );
5867 assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) );
5868 if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break;
5869 elistFlags |= pE->flags;
5870 }
5871 if( k<pEList->nExpr ){
5872 /*
5873 ** If we get here it means the result set contains one or more "*"
5874 ** operators that need to be expanded. Loop through each expression
5875 ** in the result set and expand them one by one.
5876 */
5877 struct ExprList_item *a = pEList->a;
5878 ExprList *pNew = 0;
5879 int flags = pParse->db->flags;
5880 int longNames = (flags & SQLITE_FullColNames)!=0
5881 && (flags & SQLITE_ShortColNames)==0;
5882
5883 for(k=0; k<pEList->nExpr; k++){
5884 pE = a[k].pExpr;
5885 elistFlags |= pE->flags;
5886 pRight = pE->pRight;
5887 assert( pE->op!=TK_DOT || pRight!=0 );
5888 if( pE->op!=TK_ASTERISK
5889 && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK)
5890 ){
5891 /* This particular expression does not need to be expanded.
5892 */
5893 pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr);
5894 if( pNew ){
5895 pNew->a[pNew->nExpr-1].zEName = a[k].zEName;
5896 pNew->a[pNew->nExpr-1].fg.eEName = a[k].fg.eEName;
5897 a[k].zEName = 0;
5898 }
5899 a[k].pExpr = 0;
5900 }else{
5901 /* This expression is a "*" or a "TABLE.*" and needs to be
5902 ** expanded. */
5903 int tableSeen = 0; /* Set to 1 when TABLE matches */
5904 char *zTName = 0; /* text of name of TABLE */
5905 if( pE->op==TK_DOT ){
5906 assert( pE->pLeft!=0 );
5907 assert( !ExprHasProperty(pE->pLeft, EP_IntValue) );
5908 zTName = pE->pLeft->u.zToken;
5909 }
5910 for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
5911 Table *pTab = pFrom->pTab; /* Table for this data source */
5912 ExprList *pNestedFrom; /* Result-set of a nested FROM clause */
5913 char *zTabName; /* AS name for this data source */
5914 const char *zSchemaName = 0; /* Schema name for this data source */
5915 int iDb; /* Schema index for this data src */
5916 IdList *pUsing; /* USING clause for pFrom[1] */
5917
5918 if( (zTabName = pFrom->zAlias)==0 ){
5919 zTabName = pTab->zName;
5920 }
5921 if( db->mallocFailed ) break;
5922 assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) );
5923 if( pFrom->fg.isNestedFrom ){
5924 assert( pFrom->pSelect!=0 );
5925 pNestedFrom = pFrom->pSelect->pEList;
5926 assert( pNestedFrom!=0 );
5927 assert( pNestedFrom->nExpr==pTab->nCol );
5928 }else{
5929 if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){
5930 continue;
5931 }
5932 pNestedFrom = 0;
5933 iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
5934 zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*";
5935 }
5936 if( i+1<pTabList->nSrc
5937 && pFrom[1].fg.isUsing
5938 && (selFlags & SF_NestedFrom)!=0
5939 ){
5940 int ii;
5941 pUsing = pFrom[1].u3.pUsing;
5942 for(ii=0; ii<pUsing->nId; ii++){
5943 const char *zUName = pUsing->a[ii].zName;
5944 pRight = sqlite3Expr(db, TK_ID, zUName);
5945 pNew = sqlite3ExprListAppend(pParse, pNew, pRight);
5946 if( pNew ){
5947 struct ExprList_item *pX = &pNew->a[pNew->nExpr-1];
5948 assert( pX->zEName==0 );
5949 pX->zEName = sqlite3MPrintf(db,"..%s", zUName);
5950 pX->fg.eEName = ENAME_TAB;
5951 pX->fg.bUsingTerm = 1;
5952 }
5953 }
5954 }else{
5955 pUsing = 0;
5956 }
5957 for(j=0; j<pTab->nCol; j++){
5958 char *zName = pTab->aCol[j].zCnName;
5959 struct ExprList_item *pX; /* Newly added ExprList term */
5960
5961 assert( zName );
5962 if( zTName
5963 && pNestedFrom
5964 && sqlite3MatchEName(&pNestedFrom->a[j], 0, zTName, 0)==0
5965 ){
5966 continue;
5967 }
5968
5969 /* If a column is marked as 'hidden', omit it from the expanded
5970 ** result-set list unless the SELECT has the SF_IncludeHidden
5971 ** bit set.
5972 */
5973 if( (p->selFlags & SF_IncludeHidden)==0
5974 && IsHiddenColumn(&pTab->aCol[j])
5975 ){
5976 continue;
5977 }
5978 if( (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0
5979 && zTName==0
5980 && (selFlags & (SF_NestedFrom))==0
5981 ){
5982 continue;
5983 }
5984 tableSeen = 1;
5985
5986 if( i>0 && zTName==0 && (selFlags & SF_NestedFrom)==0 ){
5987 if( pFrom->fg.isUsing
5988 && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0
5989 ){
5990 /* In a join with a USING clause, omit columns in the
5991 ** using clause from the table on the right. */
5992 continue;
5993 }
5994 }
5995 pRight = sqlite3Expr(db, TK_ID, zName);
5996 if( (pTabList->nSrc>1
5997 && ( (pFrom->fg.jointype & JT_LTORJ)==0
5998 || (selFlags & SF_NestedFrom)!=0
5999 || !inAnyUsingClause(zName,pFrom,pTabList->nSrc-i-1)
6000 )
6001 )
6002 || IN_RENAME_OBJECT
6003 ){
6004 Expr *pLeft;
6005 pLeft = sqlite3Expr(db, TK_ID, zTabName);
6006 pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight);
6007 if( IN_RENAME_OBJECT && pE->pLeft ){
6008 sqlite3RenameTokenRemap(pParse, pLeft, pE->pLeft);
6009 }
6010 if( zSchemaName ){
6011 pLeft = sqlite3Expr(db, TK_ID, zSchemaName);
6012 pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr);
6013 }
6014 }else{
6015 pExpr = pRight;
6016 }
6017 pNew = sqlite3ExprListAppend(pParse, pNew, pExpr);
6018 if( pNew==0 ){
6019 break; /* OOM */
6020 }
6021 pX = &pNew->a[pNew->nExpr-1];
6022 assert( pX->zEName==0 );
6023 if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){
6024 if( pNestedFrom ){
6025 pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName);
6026 testcase( pX->zEName==0 );
6027 }else{
6028 pX->zEName = sqlite3MPrintf(db, "%s.%s.%s",
6029 zSchemaName, zTabName, zName);
6030 testcase( pX->zEName==0 );
6031 }
6032 pX->fg.eEName = ENAME_TAB;
6033 if( (pFrom->fg.isUsing
6034 && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0)
6035 || (pUsing && sqlite3IdListIndex(pUsing, zName)>=0)
6036 || (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0
6037 ){
6038 pX->fg.bNoExpand = 1;
6039 }
6040 }else if( longNames ){
6041 pX->zEName = sqlite3MPrintf(db, "%s.%s", zTabName, zName);
6042 pX->fg.eEName = ENAME_NAME;
6043 }else{
6044 pX->zEName = sqlite3DbStrDup(db, zName);
6045 pX->fg.eEName = ENAME_NAME;
6046 }
6047 }
6048 }
6049 if( !tableSeen ){
6050 if( zTName ){
6051 sqlite3ErrorMsg(pParse, "no such table: %s", zTName);
6052 }else{
6053 sqlite3ErrorMsg(pParse, "no tables specified");
6054 }
6055 }
6056 }
6057 }
6058 sqlite3ExprListDelete(db, pEList);
6059 p->pEList = pNew;
6060 }
6061 if( p->pEList ){
6062 if( p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){
6063 sqlite3ErrorMsg(pParse, "too many columns in result set");
6064 return WRC_Abort;
6065 }
6066 if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){
6067 p->selFlags |= SF_ComplexResult;
6068 }
6069 }
6070#if TREETRACE_ENABLED
6071 if( sqlite3TreeTrace & 0x100 ){
6072 SELECTTRACE(0x100,pParse,p,("After result-set wildcard expansion:\n"));
6073 sqlite3TreeViewSelect(0, p, 0);
6074 }
6075#endif
6076 return WRC_Continue;
6077}
6078
6079#if SQLITE_DEBUG
6080/*
6081** Always assert. This xSelectCallback2 implementation proves that the
6082** xSelectCallback2 is never invoked.
6083*/
6084void sqlite3SelectWalkAssert2(Walker *NotUsed, Select *NotUsed2){
6085 UNUSED_PARAMETER2(NotUsed, NotUsed2);
6086 assert( 0 );
6087}
6088#endif
6089/*
6090** This routine "expands" a SELECT statement and all of its subqueries.
6091** For additional information on what it means to "expand" a SELECT
6092** statement, see the comment on the selectExpand worker callback above.
6093**
6094** Expanding a SELECT statement is the first step in processing a
6095** SELECT statement. The SELECT statement must be expanded before
6096** name resolution is performed.
6097**
6098** If anything goes wrong, an error message is written into pParse.
6099** The calling function can detect the problem by looking at pParse->nErr
6100** and/or pParse->db->mallocFailed.
6101*/
6102static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){
6103 Walker w;
6104 w.xExprCallback = sqlite3ExprWalkNoop;
6105 w.pParse = pParse;
6106 if( OK_IF_ALWAYS_TRUE(pParse->hasCompound) ){
6107 w.xSelectCallback = convertCompoundSelectToSubquery;
6108 w.xSelectCallback2 = 0;
6109 sqlite3WalkSelect(&w, pSelect);
6110 }
6111 w.xSelectCallback = selectExpander;
6112 w.xSelectCallback2 = sqlite3SelectPopWith;
6113 w.eCode = 0;
6114 sqlite3WalkSelect(&w, pSelect);
6115}
6116
6117
6118#ifndef SQLITE_OMIT_SUBQUERY
6119/*
6120** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo()
6121** interface.
6122**
6123** For each FROM-clause subquery, add Column.zType and Column.zColl
6124** information to the Table structure that represents the result set
6125** of that subquery.
6126**
6127** The Table structure that represents the result set was constructed
6128** by selectExpander() but the type and collation information was omitted
6129** at that point because identifiers had not yet been resolved. This
6130** routine is called after identifier resolution.
6131*/
6132static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){
6133 Parse *pParse;
6134 int i;
6135 SrcList *pTabList;
6136 SrcItem *pFrom;
6137
6138 assert( p->selFlags & SF_Resolved );
6139 if( p->selFlags & SF_HasTypeInfo ) return;
6140 p->selFlags |= SF_HasTypeInfo;
6141 pParse = pWalker->pParse;
6142 pTabList = p->pSrc;
6143 for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
6144 Table *pTab = pFrom->pTab;
6145 assert( pTab!=0 );
6146 if( (pTab->tabFlags & TF_Ephemeral)!=0 ){
6147 /* A sub-query in the FROM clause of a SELECT */
6148 Select *pSel = pFrom->pSelect;
6149 if( pSel ){
6150 while( pSel->pPrior ) pSel = pSel->pPrior;
6151 sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel,
6152 SQLITE_AFF_NONE);
6153 }
6154 }
6155 }
6156}
6157#endif
6158
6159
6160/*
6161** This routine adds datatype and collating sequence information to
6162** the Table structures of all FROM-clause subqueries in a
6163** SELECT statement.
6164**
6165** Use this routine after name resolution.
6166*/
6167static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){
6168#ifndef SQLITE_OMIT_SUBQUERY
6169 Walker w;
6170 w.xSelectCallback = sqlite3SelectWalkNoop;
6171 w.xSelectCallback2 = selectAddSubqueryTypeInfo;
6172 w.xExprCallback = sqlite3ExprWalkNoop;
6173 w.pParse = pParse;
6174 sqlite3WalkSelect(&w, pSelect);
6175#endif
6176}
6177
6178
6179/*
6180** This routine sets up a SELECT statement for processing. The
6181** following is accomplished:
6182**
6183** * VDBE Cursor numbers are assigned to all FROM-clause terms.
6184** * Ephemeral Table objects are created for all FROM-clause subqueries.
6185** * ON and USING clauses are shifted into WHERE statements
6186** * Wildcards "*" and "TABLE.*" in result sets are expanded.
6187** * Identifiers in expression are matched to tables.
6188**
6189** This routine acts recursively on all subqueries within the SELECT.
6190*/
6191void sqlite3SelectPrep(
6192 Parse *pParse, /* The parser context */
6193 Select *p, /* The SELECT statement being coded. */
6194 NameContext *pOuterNC /* Name context for container */
6195){
6196 assert( p!=0 || pParse->db->mallocFailed );
6197 assert( pParse->db->pParse==pParse );
6198 if( pParse->db->mallocFailed ) return;
6199 if( p->selFlags & SF_HasTypeInfo ) return;
6200 sqlite3SelectExpand(pParse, p);
6201 if( pParse->nErr ) return;
6202 sqlite3ResolveSelectNames(pParse, p, pOuterNC);
6203 if( pParse->nErr ) return;
6204 sqlite3SelectAddTypeInfo(pParse, p);
6205}
6206
6207/*
6208** Reset the aggregate accumulator.
6209**
6210** The aggregate accumulator is a set of memory cells that hold
6211** intermediate results while calculating an aggregate. This
6212** routine generates code that stores NULLs in all of those memory
6213** cells.
6214*/
6215static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){
6216 Vdbe *v = pParse->pVdbe;
6217 int i;
6218 struct AggInfo_func *pFunc;
6219 int nReg = pAggInfo->nFunc + pAggInfo->nColumn;
6220 assert( pParse->db->pParse==pParse );
6221 assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 );
6222 if( nReg==0 ) return;
6223 if( pParse->nErr ) return;
6224#ifdef SQLITE_DEBUG
6225 /* Verify that all AggInfo registers are within the range specified by
6226 ** AggInfo.mnReg..AggInfo.mxReg */
6227 assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 );
6228 for(i=0; i<pAggInfo->nColumn; i++){
6229 assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg
6230 && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg );
6231 }
6232 for(i=0; i<pAggInfo->nFunc; i++){
6233 assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg
6234 && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg );
6235 }
6236#endif
6237 sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg);
6238 for(pFunc=pAggInfo->aFunc, i=0; i<pAggInfo->nFunc; i++, pFunc++){
6239 if( pFunc->iDistinct>=0 ){
6240 Expr *pE = pFunc->pFExpr;
6241 assert( ExprUseXList(pE) );
6242 if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){
6243 sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one "
6244 "argument");
6245 pFunc->iDistinct = -1;
6246 }else{
6247 KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0);
6248 pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
6249 pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO);
6250 ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(DISTINCT)",
6251 pFunc->pFunc->zName));
6252 }
6253 }
6254 }
6255}
6256
6257/*
6258** Invoke the OP_AggFinalize opcode for every aggregate function
6259** in the AggInfo structure.
6260*/
6261static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
6262 Vdbe *v = pParse->pVdbe;
6263 int i;
6264 struct AggInfo_func *pF;
6265 for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){
6266 ExprList *pList;
6267 assert( ExprUseXList(pF->pFExpr) );
6268 pList = pF->pFExpr->x.pList;
6269 sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0);
6270 sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
6271 }
6272}
6273
6274
6275/*
6276** Update the accumulator memory cells for an aggregate based on
6277** the current cursor position.
6278**
6279** If regAcc is non-zero and there are no min() or max() aggregates
6280** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator
6281** registers if register regAcc contains 0. The caller will take care
6282** of setting and clearing regAcc.
6283*/
6284static void updateAccumulator(
6285 Parse *pParse,
6286 int regAcc,
6287 AggInfo *pAggInfo,
6288 int eDistinctType
6289){
6290 Vdbe *v = pParse->pVdbe;
6291 int i;
6292 int regHit = 0;
6293 int addrHitTest = 0;
6294 struct AggInfo_func *pF;
6295 struct AggInfo_col *pC;
6296
6297 pAggInfo->directMode = 1;
6298 for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){
6299 int nArg;
6300 int addrNext = 0;
6301 int regAgg;
6302 ExprList *pList;
6303 assert( ExprUseXList(pF->pFExpr) );
6304 assert( !IsWindowFunc(pF->pFExpr) );
6305 pList = pF->pFExpr->x.pList;
6306 if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){
6307 Expr *pFilter = pF->pFExpr->y.pWin->pFilter;
6308 if( pAggInfo->nAccumulator
6309 && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
6310 && regAcc
6311 ){
6312 /* If regAcc==0, there there exists some min() or max() function
6313 ** without a FILTER clause that will ensure the magnet registers
6314 ** are populated. */
6315 if( regHit==0 ) regHit = ++pParse->nMem;
6316 /* If this is the first row of the group (regAcc contains 0), clear the
6317 ** "magnet" register regHit so that the accumulator registers
6318 ** are populated if the FILTER clause jumps over the the
6319 ** invocation of min() or max() altogether. Or, if this is not
6320 ** the first row (regAcc contains 1), set the magnet register so that
6321 ** the accumulators are not populated unless the min()/max() is invoked
6322 ** and indicates that they should be. */
6323 sqlite3VdbeAddOp2(v, OP_Copy, regAcc, regHit);
6324 }
6325 addrNext = sqlite3VdbeMakeLabel(pParse);
6326 sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL);
6327 }
6328 if( pList ){
6329 nArg = pList->nExpr;
6330 regAgg = sqlite3GetTempRange(pParse, nArg);
6331 sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP);
6332 }else{
6333 nArg = 0;
6334 regAgg = 0;
6335 }
6336 if( pF->iDistinct>=0 && pList ){
6337 if( addrNext==0 ){
6338 addrNext = sqlite3VdbeMakeLabel(pParse);
6339 }
6340 pF->iDistinct = codeDistinct(pParse, eDistinctType,
6341 pF->iDistinct, addrNext, pList, regAgg);
6342 }
6343 if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){
6344 CollSeq *pColl = 0;
6345 struct ExprList_item *pItem;
6346 int j;
6347 assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */
6348 for(j=0, pItem=pList->a; !pColl && j<nArg; j++, pItem++){
6349 pColl = sqlite3ExprCollSeq(pParse, pItem->pExpr);
6350 }
6351 if( !pColl ){
6352 pColl = pParse->db->pDfltColl;
6353 }
6354 if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem;
6355 sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ);
6356 }
6357 sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem);
6358 sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
6359 sqlite3VdbeChangeP5(v, (u8)nArg);
6360 sqlite3ReleaseTempRange(pParse, regAgg, nArg);
6361 if( addrNext ){
6362 sqlite3VdbeResolveLabel(v, addrNext);
6363 }
6364 }
6365 if( regHit==0 && pAggInfo->nAccumulator ){
6366 regHit = regAcc;
6367 }
6368 if( regHit ){
6369 addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v);
6370 }
6371 for(i=0, pC=pAggInfo->aCol; i<pAggInfo->nAccumulator; i++, pC++){
6372 sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem);
6373 }
6374
6375 pAggInfo->directMode = 0;
6376 if( addrHitTest ){
6377 sqlite3VdbeJumpHereOrPopInst(v, addrHitTest);
6378 }
6379}
6380
6381/*
6382** Add a single OP_Explain instruction to the VDBE to explain a simple
6383** count(*) query ("SELECT count(*) FROM pTab").
6384*/
6385#ifndef SQLITE_OMIT_EXPLAIN
6386static void explainSimpleCount(
6387 Parse *pParse, /* Parse context */
6388 Table *pTab, /* Table being queried */
6389 Index *pIdx /* Index used to optimize scan, or NULL */
6390){
6391 if( pParse->explain==2 ){
6392 int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx)));
6393 sqlite3VdbeExplain(pParse, 0, "SCAN %s%s%s",
6394 pTab->zName,
6395 bCover ? " USING COVERING INDEX " : "",
6396 bCover ? pIdx->zName : ""
6397 );
6398 }
6399}
6400#else
6401# define explainSimpleCount(a,b,c)
6402#endif
6403
6404/*
6405** sqlite3WalkExpr() callback used by havingToWhere().
6406**
6407** If the node passed to the callback is a TK_AND node, return
6408** WRC_Continue to tell sqlite3WalkExpr() to iterate through child nodes.
6409**
6410** Otherwise, return WRC_Prune. In this case, also check if the
6411** sub-expression matches the criteria for being moved to the WHERE
6412** clause. If so, add it to the WHERE clause and replace the sub-expression
6413** within the HAVING expression with a constant "1".
6414*/
6415static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){
6416 if( pExpr->op!=TK_AND ){
6417 Select *pS = pWalker->u.pSelect;
6418 /* This routine is called before the HAVING clause of the current
6419 ** SELECT is analyzed for aggregates. So if pExpr->pAggInfo is set
6420 ** here, it indicates that the expression is a correlated reference to a
6421 ** column from an outer aggregate query, or an aggregate function that
6422 ** belongs to an outer query. Do not move the expression to the WHERE
6423 ** clause in this obscure case, as doing so may corrupt the outer Select
6424 ** statements AggInfo structure. */
6425 if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy)
6426 && ExprAlwaysFalse(pExpr)==0
6427 && pExpr->pAggInfo==0
6428 ){
6429 sqlite3 *db = pWalker->pParse->db;
6430 Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1");
6431 if( pNew ){
6432 Expr *pWhere = pS->pWhere;
6433 SWAP(Expr, *pNew, *pExpr);
6434 pNew = sqlite3ExprAnd(pWalker->pParse, pWhere, pNew);
6435 pS->pWhere = pNew;
6436 pWalker->eCode = 1;
6437 }
6438 }
6439 return WRC_Prune;
6440 }
6441 return WRC_Continue;
6442}
6443
6444/*
6445** Transfer eligible terms from the HAVING clause of a query, which is
6446** processed after grouping, to the WHERE clause, which is processed before
6447** grouping. For example, the query:
6448**
6449** SELECT * FROM <tables> WHERE a=? GROUP BY b HAVING b=? AND c=?
6450**
6451** can be rewritten as:
6452**
6453** SELECT * FROM <tables> WHERE a=? AND b=? GROUP BY b HAVING c=?
6454**
6455** A term of the HAVING expression is eligible for transfer if it consists
6456** entirely of constants and expressions that are also GROUP BY terms that
6457** use the "BINARY" collation sequence.
6458*/
6459static void havingToWhere(Parse *pParse, Select *p){
6460 Walker sWalker;
6461 memset(&sWalker, 0, sizeof(sWalker));
6462 sWalker.pParse = pParse;
6463 sWalker.xExprCallback = havingToWhereExprCb;
6464 sWalker.u.pSelect = p;
6465 sqlite3WalkExpr(&sWalker, p->pHaving);
6466#if TREETRACE_ENABLED
6467 if( sWalker.eCode && (sqlite3TreeTrace & 0x100)!=0 ){
6468 SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n"));
6469 sqlite3TreeViewSelect(0, p, 0);
6470 }
6471#endif
6472}
6473
6474/*
6475** Check to see if the pThis entry of pTabList is a self-join of a prior view.
6476** If it is, then return the SrcItem for the prior view. If it is not,
6477** then return 0.
6478*/
6479static SrcItem *isSelfJoinView(
6480 SrcList *pTabList, /* Search for self-joins in this FROM clause */
6481 SrcItem *pThis /* Search for prior reference to this subquery */
6482){
6483 SrcItem *pItem;
6484 assert( pThis->pSelect!=0 );
6485 if( pThis->pSelect->selFlags & SF_PushDown ) return 0;
6486 for(pItem = pTabList->a; pItem<pThis; pItem++){
6487 Select *pS1;
6488 if( pItem->pSelect==0 ) continue;
6489 if( pItem->fg.viaCoroutine ) continue;
6490 if( pItem->zName==0 ) continue;
6491 assert( pItem->pTab!=0 );
6492 assert( pThis->pTab!=0 );
6493 if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue;
6494 if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue;
6495 pS1 = pItem->pSelect;
6496 if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){
6497 /* The query flattener left two different CTE tables with identical
6498 ** names in the same FROM clause. */
6499 continue;
6500 }
6501 if( pItem->pSelect->selFlags & SF_PushDown ){
6502 /* The view was modified by some other optimization such as
6503 ** pushDownWhereTerms() */
6504 continue;
6505 }
6506 return pItem;
6507 }
6508 return 0;
6509}
6510
6511/*
6512** Deallocate a single AggInfo object
6513*/
6514static void agginfoFree(sqlite3 *db, AggInfo *p){
6515 sqlite3DbFree(db, p->aCol);
6516 sqlite3DbFree(db, p->aFunc);
6517 sqlite3DbFreeNN(db, p);
6518}
6519
6520#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION
6521/*
6522** Attempt to transform a query of the form
6523**
6524** SELECT count(*) FROM (SELECT x FROM t1 UNION ALL SELECT y FROM t2)
6525**
6526** Into this:
6527**
6528** SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2)
6529**
6530** The transformation only works if all of the following are true:
6531**
6532** * The subquery is a UNION ALL of two or more terms
6533** * The subquery does not have a LIMIT clause
6534** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries
6535** * The outer query is a simple count(*) with no WHERE clause or other
6536** extraneous syntax.
6537**
6538** Return TRUE if the optimization is undertaken.
6539*/
6540static int countOfViewOptimization(Parse *pParse, Select *p){
6541 Select *pSub, *pPrior;
6542 Expr *pExpr;
6543 Expr *pCount;
6544 sqlite3 *db;
6545 if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */
6546 if( p->pEList->nExpr!=1 ) return 0; /* Single result column */
6547 if( p->pWhere ) return 0;
6548 if( p->pGroupBy ) return 0;
6549 pExpr = p->pEList->a[0].pExpr;
6550 if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */
6551 assert( ExprUseUToken(pExpr) );
6552 if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0; /* Is count() */
6553 assert( ExprUseXList(pExpr) );
6554 if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */
6555 if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */
6556 pSub = p->pSrc->a[0].pSelect;
6557 if( pSub==0 ) return 0; /* The FROM is a subquery */
6558 if( pSub->pPrior==0 ) return 0; /* Must be a compound ry */
6559 do{
6560 if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */
6561 if( pSub->pWhere ) return 0; /* No WHERE clause */
6562 if( pSub->pLimit ) return 0; /* No LIMIT clause */
6563 if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */
6564 pSub = pSub->pPrior; /* Repeat over compound */
6565 }while( pSub );
6566
6567 /* If we reach this point then it is OK to perform the transformation */
6568
6569 db = pParse->db;
6570 pCount = pExpr;
6571 pExpr = 0;
6572 pSub = p->pSrc->a[0].pSelect;
6573 p->pSrc->a[0].pSelect = 0;
6574 sqlite3SrcListDelete(db, p->pSrc);
6575 p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc));
6576 while( pSub ){
6577 Expr *pTerm;
6578 pPrior = pSub->pPrior;
6579 pSub->pPrior = 0;
6580 pSub->pNext = 0;
6581 pSub->selFlags |= SF_Aggregate;
6582 pSub->selFlags &= ~SF_Compound;
6583 pSub->nSelectRow = 0;
6584 sqlite3ExprListDelete(db, pSub->pEList);
6585 pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount;
6586 pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm);
6587 pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0);
6588 sqlite3PExprAddSelect(pParse, pTerm, pSub);
6589 if( pExpr==0 ){
6590 pExpr = pTerm;
6591 }else{
6592 pExpr = sqlite3PExpr(pParse, TK_PLUS, pTerm, pExpr);
6593 }
6594 pSub = pPrior;
6595 }
6596 p->pEList->a[0].pExpr = pExpr;
6597 p->selFlags &= ~SF_Aggregate;
6598
6599#if TREETRACE_ENABLED
6600 if( sqlite3TreeTrace & 0x400 ){
6601 SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n"));
6602 sqlite3TreeViewSelect(0, p, 0);
6603 }
6604#endif
6605 return 1;
6606}
6607#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */
6608
6609/*
6610** If any term of pSrc, or any SF_NestedFrom sub-query, is not the same
6611** as pSrcItem but has the same alias as p0, then return true.
6612** Otherwise return false.
6613*/
6614static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){
6615 int i;
6616 for(i=0; i<pSrc->nSrc; i++){
6617 SrcItem *p1 = &pSrc->a[i];
6618 if( p1==p0 ) continue;
6619 if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){
6620 return 1;
6621 }
6622 if( p1->pSelect
6623 && (p1->pSelect->selFlags & SF_NestedFrom)!=0
6624 && sameSrcAlias(p0, p1->pSelect->pSrc)
6625 ){
6626 return 1;
6627 }
6628 }
6629 return 0;
6630}
6631
6632/*
6633** Generate code for the SELECT statement given in the p argument.
6634**
6635** The results are returned according to the SelectDest structure.
6636** See comments in sqliteInt.h for further information.
6637**
6638** This routine returns the number of errors. If any errors are
6639** encountered, then an appropriate error message is left in
6640** pParse->zErrMsg.
6641**
6642** This routine does NOT free the Select structure passed in. The
6643** calling function needs to do that.
6644*/
6645int sqlite3Select(
6646 Parse *pParse, /* The parser context */
6647 Select *p, /* The SELECT statement being coded. */
6648 SelectDest *pDest /* What to do with the query results */
6649){
6650 int i, j; /* Loop counters */
6651 WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */
6652 Vdbe *v; /* The virtual machine under construction */
6653 int isAgg; /* True for select lists like "count(*)" */
6654 ExprList *pEList = 0; /* List of columns to extract. */
6655 SrcList *pTabList; /* List of tables to select from */
6656 Expr *pWhere; /* The WHERE clause. May be NULL */
6657 ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */
6658 Expr *pHaving; /* The HAVING clause. May be NULL */
6659 AggInfo *pAggInfo = 0; /* Aggregate information */
6660 int rc = 1; /* Value to return from this function */
6661 DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */
6662 SortCtx sSort; /* Info on how to code the ORDER BY clause */
6663 int iEnd; /* Address of the end of the query */
6664 sqlite3 *db; /* The database connection */
6665 ExprList *pMinMaxOrderBy = 0; /* Added ORDER BY for min/max queries */
6666 u8 minMaxFlag; /* Flag for min/max queries */
6667
6668 db = pParse->db;
6669 assert( pParse==db->pParse );
6670 v = sqlite3GetVdbe(pParse);
6671 if( p==0 || pParse->nErr ){
6672 return 1;
6673 }
6674 assert( db->mallocFailed==0 );
6675 if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1;
6676#if TREETRACE_ENABLED
6677 SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain));
6678 if( sqlite3TreeTrace & 0x10100 ){
6679 if( (sqlite3TreeTrace & 0x10001)==0x10000 ){
6680 sqlite3TreeViewLine(0, "In sqlite3Select() at %s:%d",
6681 __FILE__, __LINE__);
6682 }
6683 sqlite3ShowSelect(p);
6684 }
6685#endif
6686
6687 assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo );
6688 assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo );
6689 assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue );
6690 assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue );
6691 if( IgnorableDistinct(pDest) ){
6692 assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union ||
6693 pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard ||
6694 pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo );
6695 /* All of these destinations are also able to ignore the ORDER BY clause */
6696 if( p->pOrderBy ){
6697#if TREETRACE_ENABLED
6698 SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n"));
6699 if( sqlite3TreeTrace & 0x100 ){
6700 sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY");
6701 }
6702#endif
6703 sqlite3ParserAddCleanup(pParse,
6704 (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
6705 p->pOrderBy);
6706 testcase( pParse->earlyCleanup );
6707 p->pOrderBy = 0;
6708 }
6709 p->selFlags &= ~SF_Distinct;
6710 p->selFlags |= SF_NoopOrderBy;
6711 }
6712 sqlite3SelectPrep(pParse, p, 0);
6713 if( pParse->nErr ){
6714 goto select_end;
6715 }
6716 assert( db->mallocFailed==0 );
6717 assert( p->pEList!=0 );
6718#if TREETRACE_ENABLED
6719 if( sqlite3TreeTrace & 0x104 ){
6720 SELECTTRACE(0x104,pParse,p, ("after name resolution:\n"));
6721 sqlite3TreeViewSelect(0, p, 0);
6722 }
6723#endif
6724
6725 /* If the SF_UFSrcCheck flag is set, then this function is being called
6726 ** as part of populating the temp table for an UPDATE...FROM statement.
6727 ** In this case, it is an error if the target object (pSrc->a[0]) name
6728 ** or alias is duplicated within FROM clause (pSrc->a[1..n]).
6729 **
6730 ** Postgres disallows this case too. The reason is that some other
6731 ** systems handle this case differently, and not all the same way,
6732 ** which is just confusing. To avoid this, we follow PG's lead and
6733 ** disallow it altogether. */
6734 if( p->selFlags & SF_UFSrcCheck ){
6735 SrcItem *p0 = &p->pSrc->a[0];
6736 if( sameSrcAlias(p0, p->pSrc) ){
6737 sqlite3ErrorMsg(pParse,
6738 "target object/alias may not appear in FROM clause: %s",
6739 p0->zAlias ? p0->zAlias : p0->pTab->zName
6740 );
6741 goto select_end;
6742 }
6743
6744 /* Clear the SF_UFSrcCheck flag. The check has already been performed,
6745 ** and leaving this flag set can cause errors if a compound sub-query
6746 ** in p->pSrc is flattened into this query and this function called
6747 ** again as part of compound SELECT processing. */
6748 p->selFlags &= ~SF_UFSrcCheck;
6749 }
6750
6751 if( pDest->eDest==SRT_Output ){
6752 sqlite3GenerateColumnNames(pParse, p);
6753 }
6754
6755#ifndef SQLITE_OMIT_WINDOWFUNC
6756 if( sqlite3WindowRewrite(pParse, p) ){
6757 assert( pParse->nErr );
6758 goto select_end;
6759 }
6760#if TREETRACE_ENABLED
6761 if( p->pWin && (sqlite3TreeTrace & 0x108)!=0 ){
6762 SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n"));
6763 sqlite3TreeViewSelect(0, p, 0);
6764 }
6765#endif
6766#endif /* SQLITE_OMIT_WINDOWFUNC */
6767 pTabList = p->pSrc;
6768 isAgg = (p->selFlags & SF_Aggregate)!=0;
6769 memset(&sSort, 0, sizeof(sSort));
6770 sSort.pOrderBy = p->pOrderBy;
6771
6772 /* Try to do various optimizations (flattening subqueries, and strength
6773 ** reduction of join operators) in the FROM clause up into the main query
6774 */
6775#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
6776 for(i=0; !p->pPrior && i<pTabList->nSrc; i++){
6777 SrcItem *pItem = &pTabList->a[i];
6778 Select *pSub = pItem->pSelect;
6779 Table *pTab = pItem->pTab;
6780
6781 /* The expander should have already created transient Table objects
6782 ** even for FROM clause elements such as subqueries that do not correspond
6783 ** to a real table */
6784 assert( pTab!=0 );
6785
6786 /* Convert LEFT JOIN into JOIN if there are terms of the right table
6787 ** of the LEFT JOIN used in the WHERE clause.
6788 */
6789 if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==JT_LEFT
6790 && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor)
6791 && OptimizationEnabled(db, SQLITE_SimplifyJoin)
6792 ){
6793 SELECTTRACE(0x100,pParse,p,
6794 ("LEFT-JOIN simplifies to JOIN on term %d\n",i));
6795 pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER);
6796 assert( pItem->iCursor>=0 );
6797 unsetJoinExpr(p->pWhere, pItem->iCursor,
6798 pTabList->a[0].fg.jointype & JT_LTORJ);
6799 }
6800
6801 /* No futher action if this term of the FROM clause is no a subquery */
6802 if( pSub==0 ) continue;
6803
6804 /* Catch mismatch in the declared columns of a view and the number of
6805 ** columns in the SELECT on the RHS */
6806 if( pTab->nCol!=pSub->pEList->nExpr ){
6807 sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d",
6808 pTab->nCol, pTab->zName, pSub->pEList->nExpr);
6809 goto select_end;
6810 }
6811
6812 /* Do not try to flatten an aggregate subquery.
6813 **
6814 ** Flattening an aggregate subquery is only possible if the outer query
6815 ** is not a join. But if the outer query is not a join, then the subquery
6816 ** will be implemented as a co-routine and there is no advantage to
6817 ** flattening in that case.
6818 */
6819 if( (pSub->selFlags & SF_Aggregate)!=0 ) continue;
6820 assert( pSub->pGroupBy==0 );
6821
6822 /* If a FROM-clause subquery has an ORDER BY clause that is not
6823 ** really doing anything, then delete it now so that it does not
6824 ** interfere with query flattening. See the discussion at
6825 ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a
6826 **
6827 ** Beware of these cases where the ORDER BY clause may not be safely
6828 ** omitted:
6829 **
6830 ** (1) There is also a LIMIT clause
6831 ** (2) The subquery was added to help with window-function
6832 ** processing
6833 ** (3) The subquery is in the FROM clause of an UPDATE
6834 ** (4) The outer query uses an aggregate function other than
6835 ** the built-in count(), min(), or max().
6836 ** (5) The ORDER BY isn't going to accomplish anything because
6837 ** one of:
6838 ** (a) The outer query has a different ORDER BY clause
6839 ** (b) The subquery is part of a join
6840 ** See forum post 062d576715d277c8
6841 */
6842 if( pSub->pOrderBy!=0
6843 && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */
6844 && pSub->pLimit==0 /* Condition (1) */
6845 && (pSub->selFlags & SF_OrderByReqd)==0 /* Condition (2) */
6846 && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */
6847 && OptimizationEnabled(db, SQLITE_OmitOrderBy)
6848 ){
6849 SELECTTRACE(0x100,pParse,p,
6850 ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1));
6851 sqlite3ParserAddCleanup(pParse,
6852 (void(*)(sqlite3*,void*))sqlite3ExprListDelete,
6853 pSub->pOrderBy);
6854 pSub->pOrderBy = 0;
6855 }
6856
6857 /* If the outer query contains a "complex" result set (that is,
6858 ** if the result set of the outer query uses functions or subqueries)
6859 ** and if the subquery contains an ORDER BY clause and if
6860 ** it will be implemented as a co-routine, then do not flatten. This
6861 ** restriction allows SQL constructs like this:
6862 **
6863 ** SELECT expensive_function(x)
6864 ** FROM (SELECT x FROM tab ORDER BY y LIMIT 10);
6865 **
6866 ** The expensive_function() is only computed on the 10 rows that
6867 ** are output, rather than every row of the table.
6868 **
6869 ** The requirement that the outer query have a complex result set
6870 ** means that flattening does occur on simpler SQL constraints without
6871 ** the expensive_function() like:
6872 **
6873 ** SELECT x FROM (SELECT x FROM tab ORDER BY y LIMIT 10);
6874 */
6875 if( pSub->pOrderBy!=0
6876 && i==0
6877 && (p->selFlags & SF_ComplexResult)!=0
6878 && (pTabList->nSrc==1
6879 || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0)
6880 ){
6881 continue;
6882 }
6883
6884 if( flattenSubquery(pParse, p, i, isAgg) ){
6885 if( pParse->nErr ) goto select_end;
6886 /* This subquery can be absorbed into its parent. */
6887 i = -1;
6888 }
6889 pTabList = p->pSrc;
6890 if( db->mallocFailed ) goto select_end;
6891 if( !IgnorableOrderby(pDest) ){
6892 sSort.pOrderBy = p->pOrderBy;
6893 }
6894 }
6895#endif
6896
6897#ifndef SQLITE_OMIT_COMPOUND_SELECT
6898 /* Handle compound SELECT statements using the separate multiSelect()
6899 ** procedure.
6900 */
6901 if( p->pPrior ){
6902 rc = multiSelect(pParse, p, pDest);
6903#if TREETRACE_ENABLED
6904 SELECTTRACE(0x1,pParse,p,("end compound-select processing\n"));
6905 if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){
6906 sqlite3TreeViewSelect(0, p, 0);
6907 }
6908#endif
6909 if( p->pNext==0 ) ExplainQueryPlanPop(pParse);
6910 return rc;
6911 }
6912#endif
6913
6914 /* Do the WHERE-clause constant propagation optimization if this is
6915 ** a join. No need to speed time on this operation for non-join queries
6916 ** as the equivalent optimization will be handled by query planner in
6917 ** sqlite3WhereBegin().
6918 */
6919 if( p->pWhere!=0
6920 && p->pWhere->op==TK_AND
6921 && OptimizationEnabled(db, SQLITE_PropagateConst)
6922 && propagateConstants(pParse, p)
6923 ){
6924#if TREETRACE_ENABLED
6925 if( sqlite3TreeTrace & 0x100 ){
6926 SELECTTRACE(0x100,pParse,p,("After constant propagation:\n"));
6927 sqlite3TreeViewSelect(0, p, 0);
6928 }
6929#endif
6930 }else{
6931 SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n"));
6932 }
6933
6934#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION
6935 if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView)
6936 && countOfViewOptimization(pParse, p)
6937 ){
6938 if( db->mallocFailed ) goto select_end;
6939 pEList = p->pEList;
6940 pTabList = p->pSrc;
6941 }
6942#endif
6943
6944 /* For each term in the FROM clause, do two things:
6945 ** (1) Authorized unreferenced tables
6946 ** (2) Generate code for all sub-queries
6947 */
6948 for(i=0; i<pTabList->nSrc; i++){
6949 SrcItem *pItem = &pTabList->a[i];
6950 SrcItem *pPrior;
6951 SelectDest dest;
6952 Select *pSub;
6953#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
6954 const char *zSavedAuthContext;
6955#endif
6956
6957 /* Issue SQLITE_READ authorizations with a fake column name for any
6958 ** tables that are referenced but from which no values are extracted.
6959 ** Examples of where these kinds of null SQLITE_READ authorizations
6960 ** would occur:
6961 **
6962 ** SELECT count(*) FROM t1; -- SQLITE_READ t1.""
6963 ** SELECT t1.* FROM t1, t2; -- SQLITE_READ t2.""
6964 **
6965 ** The fake column name is an empty string. It is possible for a table to
6966 ** have a column named by the empty string, in which case there is no way to
6967 ** distinguish between an unreferenced table and an actual reference to the
6968 ** "" column. The original design was for the fake column name to be a NULL,
6969 ** which would be unambiguous. But legacy authorization callbacks might
6970 ** assume the column name is non-NULL and segfault. The use of an empty
6971 ** string for the fake column name seems safer.
6972 */
6973 if( pItem->colUsed==0 && pItem->zName!=0 ){
6974 sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase);
6975 }
6976
6977#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
6978 /* Generate code for all sub-queries in the FROM clause
6979 */
6980 pSub = pItem->pSelect;
6981 if( pSub==0 ) continue;
6982
6983 /* The code for a subquery should only be generated once. */
6984 assert( pItem->addrFillSub==0 );
6985
6986 /* Increment Parse.nHeight by the height of the largest expression
6987 ** tree referred to by this, the parent select. The child select
6988 ** may contain expression trees of at most
6989 ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit
6990 ** more conservative than necessary, but much easier than enforcing
6991 ** an exact limit.
6992 */
6993 pParse->nHeight += sqlite3SelectExprHeight(p);
6994
6995 /* Make copies of constant WHERE-clause terms in the outer query down
6996 ** inside the subquery. This can help the subquery to run more efficiently.
6997 */
6998 if( OptimizationEnabled(db, SQLITE_PushDown)
6999 && (pItem->fg.isCte==0
7000 || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2))
7001 && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem)
7002 ){
7003#if TREETRACE_ENABLED
7004 if( sqlite3TreeTrace & 0x100 ){
7005 SELECTTRACE(0x100,pParse,p,
7006 ("After WHERE-clause push-down into subquery %d:\n", pSub->selId));
7007 sqlite3TreeViewSelect(0, p, 0);
7008 }
7009#endif
7010 assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 );
7011 }else{
7012 SELECTTRACE(0x100,pParse,p,("Push-down not possible\n"));
7013 }
7014
7015 zSavedAuthContext = pParse->zAuthContext;
7016 pParse->zAuthContext = pItem->zName;
7017
7018 /* Generate code to implement the subquery
7019 **
7020 ** The subquery is implemented as a co-routine if all of the following are
7021 ** true:
7022 **
7023 ** (1) the subquery is guaranteed to be the outer loop (so that
7024 ** it does not need to be computed more than once), and
7025 ** (2) the subquery is not a CTE that should be materialized
7026 ** (3) the subquery is not part of a left operand for a RIGHT JOIN
7027 */
7028 if( i==0
7029 && (pTabList->nSrc==1
7030 || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0) /* (1) */
7031 && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) /* (2) */
7032 && (pTabList->a[0].fg.jointype & JT_LTORJ)==0 /* (3) */
7033 ){
7034 /* Implement a co-routine that will return a single row of the result
7035 ** set on each invocation.
7036 */
7037 int addrTop = sqlite3VdbeCurrentAddr(v)+1;
7038
7039 pItem->regReturn = ++pParse->nMem;
7040 sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop);
7041 VdbeComment((v, "%!S", pItem));
7042 pItem->addrFillSub = addrTop;
7043 sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn);
7044 ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem));
7045 sqlite3Select(pParse, pSub, &dest);
7046 pItem->pTab->nRowLogEst = pSub->nSelectRow;
7047 pItem->fg.viaCoroutine = 1;
7048 pItem->regResult = dest.iSdst;
7049 sqlite3VdbeEndCoroutine(v, pItem->regReturn);
7050 sqlite3VdbeJumpHere(v, addrTop-1);
7051 sqlite3ClearTempRegCache(pParse);
7052 }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){
7053 /* This is a CTE for which materialization code has already been
7054 ** generated. Invoke the subroutine to compute the materialization,
7055 ** the make the pItem->iCursor be a copy of the ephemerial table that
7056 ** holds the result of the materialization. */
7057 CteUse *pCteUse = pItem->u2.pCteUse;
7058 sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e);
7059 if( pItem->iCursor!=pCteUse->iCur ){
7060 sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pCteUse->iCur);
7061 VdbeComment((v, "%!S", pItem));
7062 }
7063 pSub->nSelectRow = pCteUse->nRowEst;
7064 }else if( (pPrior = isSelfJoinView(pTabList, pItem))!=0 ){
7065 /* This view has already been materialized by a prior entry in
7066 ** this same FROM clause. Reuse it. */
7067 if( pPrior->addrFillSub ){
7068 sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub);
7069 }
7070 sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor);
7071 pSub->nSelectRow = pPrior->pSelect->nSelectRow;
7072 }else{
7073 /* Materialize the view. If the view is not correlated, generate a
7074 ** subroutine to do the materialization so that subsequent uses of
7075 ** the same view can reuse the materialization. */
7076 int topAddr;
7077 int onceAddr = 0;
7078
7079 pItem->regReturn = ++pParse->nMem;
7080 topAddr = sqlite3VdbeAddOp0(v, OP_Goto);
7081 pItem->addrFillSub = topAddr+1;
7082 pItem->fg.isMaterialized = 1;
7083 if( pItem->fg.isCorrelated==0 ){
7084 /* If the subquery is not correlated and if we are not inside of
7085 ** a trigger, then we only need to compute the value of the subquery
7086 ** once. */
7087 onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
7088 VdbeComment((v, "materialize %!S", pItem));
7089 }else{
7090 VdbeNoopComment((v, "materialize %!S", pItem));
7091 }
7092 sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);
7093 ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem));
7094 dest.zAffSdst = sqlite3TableAffinityStr(db, pItem->pTab);
7095 sqlite3Select(pParse, pSub, &dest);
7096 sqlite3DbFree(db, dest.zAffSdst);
7097 dest.zAffSdst = 0;
7098 pItem->pTab->nRowLogEst = pSub->nSelectRow;
7099 if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr);
7100 sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1);
7101 VdbeComment((v, "end %!S", pItem));
7102 sqlite3VdbeJumpHere(v, topAddr);
7103 sqlite3ClearTempRegCache(pParse);
7104 if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){
7105 CteUse *pCteUse = pItem->u2.pCteUse;
7106 pCteUse->addrM9e = pItem->addrFillSub;
7107 pCteUse->regRtn = pItem->regReturn;
7108 pCteUse->iCur = pItem->iCursor;
7109 pCteUse->nRowEst = pSub->nSelectRow;
7110 }
7111 }
7112 if( db->mallocFailed ) goto select_end;
7113 pParse->nHeight -= sqlite3SelectExprHeight(p);
7114 pParse->zAuthContext = zSavedAuthContext;
7115#endif
7116 }
7117
7118 /* Various elements of the SELECT copied into local variables for
7119 ** convenience */
7120 pEList = p->pEList;
7121 pWhere = p->pWhere;
7122 pGroupBy = p->pGroupBy;
7123 pHaving = p->pHaving;
7124 sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0;
7125
7126#if TREETRACE_ENABLED
7127 if( sqlite3TreeTrace & 0x400 ){
7128 SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n"));
7129 sqlite3TreeViewSelect(0, p, 0);
7130 }
7131#endif
7132
7133 /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and
7134 ** if the select-list is the same as the ORDER BY list, then this query
7135 ** can be rewritten as a GROUP BY. In other words, this:
7136 **
7137 ** SELECT DISTINCT xyz FROM ... ORDER BY xyz
7138 **
7139 ** is transformed to:
7140 **
7141 ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz
7142 **
7143 ** The second form is preferred as a single index (or temp-table) may be
7144 ** used for both the ORDER BY and DISTINCT processing. As originally
7145 ** written the query must use a temp-table for at least one of the ORDER
7146 ** BY and DISTINCT, and an index or separate temp-table for the other.
7147 */
7148 if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct
7149 && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0
7150#ifndef SQLITE_OMIT_WINDOWFUNC
7151 && p->pWin==0
7152#endif
7153 ){
7154 p->selFlags &= ~SF_Distinct;
7155 pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0);
7156 p->selFlags |= SF_Aggregate;
7157 /* Notice that even thought SF_Distinct has been cleared from p->selFlags,
7158 ** the sDistinct.isTnct is still set. Hence, isTnct represents the
7159 ** original setting of the SF_Distinct flag, not the current setting */
7160 assert( sDistinct.isTnct );
7161 sDistinct.isTnct = 2;
7162
7163#if TREETRACE_ENABLED
7164 if( sqlite3TreeTrace & 0x400 ){
7165 SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n"));
7166 sqlite3TreeViewSelect(0, p, 0);
7167 }
7168#endif
7169 }
7170
7171 /* If there is an ORDER BY clause, then create an ephemeral index to
7172 ** do the sorting. But this sorting ephemeral index might end up
7173 ** being unused if the data can be extracted in pre-sorted order.
7174 ** If that is the case, then the OP_OpenEphemeral instruction will be
7175 ** changed to an OP_Noop once we figure out that the sorting index is
7176 ** not needed. The sSort.addrSortIndex variable is used to facilitate
7177 ** that change.
7178 */
7179 if( sSort.pOrderBy ){
7180 KeyInfo *pKeyInfo;
7181 pKeyInfo = sqlite3KeyInfoFromExprList(
7182 pParse, sSort.pOrderBy, 0, pEList->nExpr);
7183 sSort.iECursor = pParse->nTab++;
7184 sSort.addrSortIndex =
7185 sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
7186 sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0,
7187 (char*)pKeyInfo, P4_KEYINFO
7188 );
7189 }else{
7190 sSort.addrSortIndex = -1;
7191 }
7192
7193 /* If the output is destined for a temporary table, open that table.
7194 */
7195 if( pDest->eDest==SRT_EphemTab ){
7196 sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr);
7197 if( p->selFlags & SF_NestedFrom ){
7198 /* Delete or NULL-out result columns that will never be used */
7199 int ii;
7200 for(ii=pEList->nExpr-1; ii>0 && pEList->a[ii].fg.bUsed==0; ii--){
7201 sqlite3ExprDelete(db, pEList->a[ii].pExpr);
7202 sqlite3DbFree(db, pEList->a[ii].zEName);
7203 pEList->nExpr--;
7204 }
7205 for(ii=0; ii<pEList->nExpr; ii++){
7206 if( pEList->a[ii].fg.bUsed==0 ) pEList->a[ii].pExpr->op = TK_NULL;
7207 }
7208 }
7209 }
7210
7211 /* Set the limiter.
7212 */
7213 iEnd = sqlite3VdbeMakeLabel(pParse);
7214 if( (p->selFlags & SF_FixedLimit)==0 ){
7215 p->nSelectRow = 320; /* 4 billion rows */
7216 }
7217 if( p->pLimit ) computeLimitRegisters(pParse, p, iEnd);
7218 if( p->iLimit==0 && sSort.addrSortIndex>=0 ){
7219 sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen);
7220 sSort.sortFlags |= SORTFLAG_UseSorter;
7221 }
7222
7223 /* Open an ephemeral index to use for the distinct set.
7224 */
7225 if( p->selFlags & SF_Distinct ){
7226 sDistinct.tabTnct = pParse->nTab++;
7227 sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
7228 sDistinct.tabTnct, 0, 0,
7229 (char*)sqlite3KeyInfoFromExprList(pParse, p->pEList,0,0),
7230 P4_KEYINFO);
7231 sqlite3VdbeChangeP5(v, BTREE_UNORDERED);
7232 sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED;
7233 }else{
7234 sDistinct.eTnctType = WHERE_DISTINCT_NOOP;
7235 }
7236
7237 if( !isAgg && pGroupBy==0 ){
7238 /* No aggregate functions and no GROUP BY clause */
7239 u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0)
7240 | (p->selFlags & SF_FixedLimit);
7241#ifndef SQLITE_OMIT_WINDOWFUNC
7242 Window *pWin = p->pWin; /* Main window object (or NULL) */
7243 if( pWin ){
7244 sqlite3WindowCodeInit(pParse, p);
7245 }
7246#endif
7247 assert( WHERE_USE_LIMIT==SF_FixedLimit );
7248
7249
7250 /* Begin the database scan. */
7251 SELECTTRACE(1,pParse,p,("WhereBegin\n"));
7252 pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy,
7253 p->pEList, p, wctrlFlags, p->nSelectRow);
7254 if( pWInfo==0 ) goto select_end;
7255 if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){
7256 p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo);
7257 }
7258 if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){
7259 sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo);
7260 }
7261 if( sSort.pOrderBy ){
7262 sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo);
7263 sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo);
7264 if( sSort.nOBSat==sSort.pOrderBy->nExpr ){
7265 sSort.pOrderBy = 0;
7266 }
7267 }
7268 SELECTTRACE(1,pParse,p,("WhereBegin returns\n"));
7269
7270 /* If sorting index that was created by a prior OP_OpenEphemeral
7271 ** instruction ended up not being needed, then change the OP_OpenEphemeral
7272 ** into an OP_Noop.
7273 */
7274 if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){
7275 sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex);
7276 }
7277
7278 assert( p->pEList==pEList );
7279#ifndef SQLITE_OMIT_WINDOWFUNC
7280 if( pWin ){
7281 int addrGosub = sqlite3VdbeMakeLabel(pParse);
7282 int iCont = sqlite3VdbeMakeLabel(pParse);
7283 int iBreak = sqlite3VdbeMakeLabel(pParse);
7284 int regGosub = ++pParse->nMem;
7285
7286 sqlite3WindowCodeStep(pParse, p, pWInfo, regGosub, addrGosub);
7287
7288 sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
7289 sqlite3VdbeResolveLabel(v, addrGosub);
7290 VdbeNoopComment((v, "inner-loop subroutine"));
7291 sSort.labelOBLopt = 0;
7292 selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, iCont, iBreak);
7293 sqlite3VdbeResolveLabel(v, iCont);
7294 sqlite3VdbeAddOp1(v, OP_Return, regGosub);
7295 VdbeComment((v, "end inner-loop subroutine"));
7296 sqlite3VdbeResolveLabel(v, iBreak);
7297 }else
7298#endif /* SQLITE_OMIT_WINDOWFUNC */
7299 {
7300 /* Use the standard inner loop. */
7301 selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest,
7302 sqlite3WhereContinueLabel(pWInfo),
7303 sqlite3WhereBreakLabel(pWInfo));
7304
7305 /* End the database scan loop.
7306 */
7307 SELECTTRACE(1,pParse,p,("WhereEnd\n"));
7308 sqlite3WhereEnd(pWInfo);
7309 }
7310 }else{
7311 /* This case when there exist aggregate functions or a GROUP BY clause
7312 ** or both */
7313 NameContext sNC; /* Name context for processing aggregate information */
7314 int iAMem; /* First Mem address for storing current GROUP BY */
7315 int iBMem; /* First Mem address for previous GROUP BY */
7316 int iUseFlag; /* Mem address holding flag indicating that at least
7317 ** one row of the input to the aggregator has been
7318 ** processed */
7319 int iAbortFlag; /* Mem address which causes query abort if positive */
7320 int groupBySort; /* Rows come from source in GROUP BY order */
7321 int addrEnd; /* End of processing for this SELECT */
7322 int sortPTab = 0; /* Pseudotable used to decode sorting results */
7323 int sortOut = 0; /* Output register from the sorter */
7324 int orderByGrp = 0; /* True if the GROUP BY and ORDER BY are the same */
7325
7326 /* Remove any and all aliases between the result set and the
7327 ** GROUP BY clause.
7328 */
7329 if( pGroupBy ){
7330 int k; /* Loop counter */
7331 struct ExprList_item *pItem; /* For looping over expression in a list */
7332
7333 for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){
7334 pItem->u.x.iAlias = 0;
7335 }
7336 for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){
7337 pItem->u.x.iAlias = 0;
7338 }
7339 assert( 66==sqlite3LogEst(100) );
7340 if( p->nSelectRow>66 ) p->nSelectRow = 66;
7341
7342 /* If there is both a GROUP BY and an ORDER BY clause and they are
7343 ** identical, then it may be possible to disable the ORDER BY clause
7344 ** on the grounds that the GROUP BY will cause elements to come out
7345 ** in the correct order. It also may not - the GROUP BY might use a
7346 ** database index that causes rows to be grouped together as required
7347 ** but not actually sorted. Either way, record the fact that the
7348 ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp
7349 ** variable. */
7350 if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){
7351 int ii;
7352 /* The GROUP BY processing doesn't care whether rows are delivered in
7353 ** ASC or DESC order - only that each group is returned contiguously.
7354 ** So set the ASC/DESC flags in the GROUP BY to match those in the
7355 ** ORDER BY to maximize the chances of rows being delivered in an
7356 ** order that makes the ORDER BY redundant. */
7357 for(ii=0; ii<pGroupBy->nExpr; ii++){
7358 u8 sortFlags;
7359 sortFlags = sSort.pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC;
7360 pGroupBy->a[ii].fg.sortFlags = sortFlags;
7361 }
7362 if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){
7363 orderByGrp = 1;
7364 }
7365 }
7366 }else{
7367 assert( 0==sqlite3LogEst(1) );
7368 p->nSelectRow = 0;
7369 }
7370
7371 /* Create a label to jump to when we want to abort the query */
7372 addrEnd = sqlite3VdbeMakeLabel(pParse);
7373
7374 /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in
7375 ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the
7376 ** SELECT statement.
7377 */
7378 pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) );
7379 if( pAggInfo ){
7380 sqlite3ParserAddCleanup(pParse,
7381 (void(*)(sqlite3*,void*))agginfoFree, pAggInfo);
7382 testcase( pParse->earlyCleanup );
7383 }
7384 if( db->mallocFailed ){
7385 goto select_end;
7386 }
7387 pAggInfo->selId = p->selId;
7388 memset(&sNC, 0, sizeof(sNC));
7389 sNC.pParse = pParse;
7390 sNC.pSrcList = pTabList;
7391 sNC.uNC.pAggInfo = pAggInfo;
7392 VVA_ONLY( sNC.ncFlags = NC_UAggInfo; )
7393 pAggInfo->mnReg = pParse->nMem+1;
7394 pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0;
7395 pAggInfo->pGroupBy = pGroupBy;
7396 sqlite3ExprAnalyzeAggList(&sNC, pEList);
7397 sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy);
7398 if( pHaving ){
7399 if( pGroupBy ){
7400 assert( pWhere==p->pWhere );
7401 assert( pHaving==p->pHaving );
7402 assert( pGroupBy==p->pGroupBy );
7403 havingToWhere(pParse, p);
7404 pWhere = p->pWhere;
7405 }
7406 sqlite3ExprAnalyzeAggregates(&sNC, pHaving);
7407 }
7408 pAggInfo->nAccumulator = pAggInfo->nColumn;
7409 if( p->pGroupBy==0 && p->pHaving==0 && pAggInfo->nFunc==1 ){
7410 minMaxFlag = minMaxQuery(db, pAggInfo->aFunc[0].pFExpr, &pMinMaxOrderBy);
7411 }else{
7412 minMaxFlag = WHERE_ORDERBY_NORMAL;
7413 }
7414 for(i=0; i<pAggInfo->nFunc; i++){
7415 Expr *pExpr = pAggInfo->aFunc[i].pFExpr;
7416 assert( ExprUseXList(pExpr) );
7417 sNC.ncFlags |= NC_InAggFunc;
7418 sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList);
7419#ifndef SQLITE_OMIT_WINDOWFUNC
7420 assert( !IsWindowFunc(pExpr) );
7421 if( ExprHasProperty(pExpr, EP_WinFunc) ){
7422 sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter);
7423 }
7424#endif
7425 sNC.ncFlags &= ~NC_InAggFunc;
7426 }
7427 pAggInfo->mxReg = pParse->nMem;
7428 if( db->mallocFailed ) goto select_end;
7429#if TREETRACE_ENABLED
7430 if( sqlite3TreeTrace & 0x400 ){
7431 int ii;
7432 SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo));
7433 sqlite3TreeViewSelect(0, p, 0);
7434 if( minMaxFlag ){
7435 sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag);
7436 sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY");
7437 }
7438 for(ii=0; ii<pAggInfo->nColumn; ii++){
7439 struct AggInfo_col *pCol = &pAggInfo->aCol[ii];
7440 sqlite3DebugPrintf(
7441 "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d"
7442 " iSorterColumn=%d\n",
7443 ii, pCol->pTab ? pCol->pTab->zName : "NULL",
7444 pCol->iTable, pCol->iColumn, pCol->iMem,
7445 pCol->iSorterColumn);
7446 sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0);
7447 }
7448 for(ii=0; ii<pAggInfo->nFunc; ii++){
7449 sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n",
7450 ii, pAggInfo->aFunc[ii].iMem);
7451 sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0);
7452 }
7453 }
7454#endif
7455
7456
7457 /* Processing for aggregates with GROUP BY is very different and
7458 ** much more complex than aggregates without a GROUP BY.
7459 */
7460 if( pGroupBy ){
7461 KeyInfo *pKeyInfo; /* Keying information for the group by clause */
7462 int addr1; /* A-vs-B comparision jump */
7463 int addrOutputRow; /* Start of subroutine that outputs a result row */
7464 int regOutputRow; /* Return address register for output subroutine */
7465 int addrSetAbort; /* Set the abort flag and return */
7466 int addrTopOfLoop; /* Top of the input loop */
7467 int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */
7468 int addrReset; /* Subroutine for resetting the accumulator */
7469 int regReset; /* Return address register for reset subroutine */
7470 ExprList *pDistinct = 0;
7471 u16 distFlag = 0;
7472 int eDist = WHERE_DISTINCT_NOOP;
7473
7474 if( pAggInfo->nFunc==1
7475 && pAggInfo->aFunc[0].iDistinct>=0
7476 && ALWAYS(pAggInfo->aFunc[0].pFExpr!=0)
7477 && ALWAYS(ExprUseXList(pAggInfo->aFunc[0].pFExpr))
7478 && pAggInfo->aFunc[0].pFExpr->x.pList!=0
7479 ){
7480 Expr *pExpr = pAggInfo->aFunc[0].pFExpr->x.pList->a[0].pExpr;
7481 pExpr = sqlite3ExprDup(db, pExpr, 0);
7482 pDistinct = sqlite3ExprListDup(db, pGroupBy, 0);
7483 pDistinct = sqlite3ExprListAppend(pParse, pDistinct, pExpr);
7484 distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0;
7485 }
7486
7487 /* If there is a GROUP BY clause we might need a sorting index to
7488 ** implement it. Allocate that sorting index now. If it turns out
7489 ** that we do not need it after all, the OP_SorterOpen instruction
7490 ** will be converted into a Noop.
7491 */
7492 pAggInfo->sortingIdx = pParse->nTab++;
7493 pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pGroupBy,
7494 0, pAggInfo->nColumn);
7495 addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen,
7496 pAggInfo->sortingIdx, pAggInfo->nSortingColumn,
7497 0, (char*)pKeyInfo, P4_KEYINFO);
7498
7499 /* Initialize memory locations used by GROUP BY aggregate processing
7500 */
7501 iUseFlag = ++pParse->nMem;
7502 iAbortFlag = ++pParse->nMem;
7503 regOutputRow = ++pParse->nMem;
7504 addrOutputRow = sqlite3VdbeMakeLabel(pParse);
7505 regReset = ++pParse->nMem;
7506 addrReset = sqlite3VdbeMakeLabel(pParse);
7507 iAMem = pParse->nMem + 1;
7508 pParse->nMem += pGroupBy->nExpr;
7509 iBMem = pParse->nMem + 1;
7510 pParse->nMem += pGroupBy->nExpr;
7511 sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag);
7512 VdbeComment((v, "clear abort flag"));
7513 sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1);
7514
7515 /* Begin a loop that will extract all source rows in GROUP BY order.
7516 ** This might involve two separate loops with an OP_Sort in between, or
7517 ** it might be a single loop that uses an index to extract information
7518 ** in the right order to begin with.
7519 */
7520 sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
7521 SELECTTRACE(1,pParse,p,("WhereBegin\n"));
7522 pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct,
7523 p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY)
7524 | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0
7525 );
7526 if( pWInfo==0 ){
7527 sqlite3ExprListDelete(db, pDistinct);
7528 goto select_end;
7529 }
7530 eDist = sqlite3WhereIsDistinct(pWInfo);
7531 SELECTTRACE(1,pParse,p,("WhereBegin returns\n"));
7532 if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){
7533 /* The optimizer is able to deliver rows in group by order so
7534 ** we do not have to sort. The OP_OpenEphemeral table will be
7535 ** cancelled later because we still need to use the pKeyInfo
7536 */
7537 groupBySort = 0;
7538 }else{
7539 /* Rows are coming out in undetermined order. We have to push
7540 ** each row into a sorting index, terminate the first loop,
7541 ** then loop over the sorting index in order to get the output
7542 ** in sorted order
7543 */
7544 int regBase;
7545 int regRecord;
7546 int nCol;
7547 int nGroupBy;
7548
7549 explainTempTable(pParse,
7550 (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ?
7551 "DISTINCT" : "GROUP BY");
7552
7553 groupBySort = 1;
7554 nGroupBy = pGroupBy->nExpr;
7555 nCol = nGroupBy;
7556 j = nGroupBy;
7557 for(i=0; i<pAggInfo->nColumn; i++){
7558 if( pAggInfo->aCol[i].iSorterColumn>=j ){
7559 nCol++;
7560 j++;
7561 }
7562 }
7563 regBase = sqlite3GetTempRange(pParse, nCol);
7564 sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0);
7565 j = nGroupBy;
7566 pAggInfo->directMode = 1;
7567 for(i=0; i<pAggInfo->nColumn; i++){
7568 struct AggInfo_col *pCol = &pAggInfo->aCol[i];
7569 if( pCol->iSorterColumn>=j ){
7570 sqlite3ExprCode(pParse, pCol->pCExpr, j + regBase);
7571 j++;
7572 }
7573 }
7574 pAggInfo->directMode = 0;
7575 regRecord = sqlite3GetTempReg(pParse);
7576 sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord);
7577 sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord);
7578 sqlite3ReleaseTempReg(pParse, regRecord);
7579 sqlite3ReleaseTempRange(pParse, regBase, nCol);
7580 SELECTTRACE(1,pParse,p,("WhereEnd\n"));
7581 sqlite3WhereEnd(pWInfo);
7582 pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++;
7583 sortOut = sqlite3GetTempReg(pParse);
7584 sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol);
7585 sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd);
7586 VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v);
7587 pAggInfo->useSortingIdx = 1;
7588 }
7589
7590 /* If the index or temporary table used by the GROUP BY sort
7591 ** will naturally deliver rows in the order required by the ORDER BY
7592 ** clause, cancel the ephemeral table open coded earlier.
7593 **
7594 ** This is an optimization - the correct answer should result regardless.
7595 ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to
7596 ** disable this optimization for testing purposes. */
7597 if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder)
7598 && (groupBySort || sqlite3WhereIsSorted(pWInfo))
7599 ){
7600 sSort.pOrderBy = 0;
7601 sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex);
7602 }
7603
7604 /* Evaluate the current GROUP BY terms and store in b0, b1, b2...
7605 ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth)
7606 ** Then compare the current GROUP BY terms against the GROUP BY terms
7607 ** from the previous row currently stored in a0, a1, a2...
7608 */
7609 addrTopOfLoop = sqlite3VdbeCurrentAddr(v);
7610 if( groupBySort ){
7611 sqlite3VdbeAddOp3(v, OP_SorterData, pAggInfo->sortingIdx,
7612 sortOut, sortPTab);
7613 }
7614 for(j=0; j<pGroupBy->nExpr; j++){
7615 if( groupBySort ){
7616 sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j);
7617 }else{
7618 pAggInfo->directMode = 1;
7619 sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j);
7620 }
7621 }
7622 sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr,
7623 (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO);
7624 addr1 = sqlite3VdbeCurrentAddr(v);
7625 sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v);
7626
7627 /* Generate code that runs whenever the GROUP BY changes.
7628 ** Changes in the GROUP BY are detected by the previous code
7629 ** block. If there were no changes, this block is skipped.
7630 **
7631 ** This code copies current group by terms in b0,b1,b2,...
7632 ** over to a0,a1,a2. It then calls the output subroutine
7633 ** and resets the aggregate accumulator registers in preparation
7634 ** for the next GROUP BY batch.
7635 */
7636 sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr);
7637 sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow);
7638 VdbeComment((v, "output one row"));
7639 sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v);
7640 VdbeComment((v, "check abort flag"));
7641 sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
7642 VdbeComment((v, "reset accumulator"));
7643
7644 /* Update the aggregate accumulators based on the content of
7645 ** the current row
7646 */
7647 sqlite3VdbeJumpHere(v, addr1);
7648 updateAccumulator(pParse, iUseFlag, pAggInfo, eDist);
7649 sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag);
7650 VdbeComment((v, "indicate data in accumulator"));
7651
7652 /* End of the loop
7653 */
7654 if( groupBySort ){
7655 sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop);
7656 VdbeCoverage(v);
7657 }else{
7658 SELECTTRACE(1,pParse,p,("WhereEnd\n"));
7659 sqlite3WhereEnd(pWInfo);
7660 sqlite3VdbeChangeToNoop(v, addrSortingIdx);
7661 }
7662 sqlite3ExprListDelete(db, pDistinct);
7663
7664 /* Output the final row of result
7665 */
7666 sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow);
7667 VdbeComment((v, "output final row"));
7668
7669 /* Jump over the subroutines
7670 */
7671 sqlite3VdbeGoto(v, addrEnd);
7672
7673 /* Generate a subroutine that outputs a single row of the result
7674 ** set. This subroutine first looks at the iUseFlag. If iUseFlag
7675 ** is less than or equal to zero, the subroutine is a no-op. If
7676 ** the processing calls for the query to abort, this subroutine
7677 ** increments the iAbortFlag memory location before returning in
7678 ** order to signal the caller to abort.
7679 */
7680 addrSetAbort = sqlite3VdbeCurrentAddr(v);
7681 sqlite3VdbeAddOp2(v, OP_Integer, 1, iAbortFlag);
7682 VdbeComment((v, "set abort flag"));
7683 sqlite3VdbeAddOp1(v, OP_Return, regOutputRow);
7684 sqlite3VdbeResolveLabel(v, addrOutputRow);
7685 addrOutputRow = sqlite3VdbeCurrentAddr(v);
7686 sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2);
7687 VdbeCoverage(v);
7688 VdbeComment((v, "Groupby result generator entry point"));
7689 sqlite3VdbeAddOp1(v, OP_Return, regOutputRow);
7690 finalizeAggFunctions(pParse, pAggInfo);
7691 sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL);
7692 selectInnerLoop(pParse, p, -1, &sSort,
7693 &sDistinct, pDest,
7694 addrOutputRow+1, addrSetAbort);
7695 sqlite3VdbeAddOp1(v, OP_Return, regOutputRow);
7696 VdbeComment((v, "end groupby result generator"));
7697
7698 /* Generate a subroutine that will reset the group-by accumulator
7699 */
7700 sqlite3VdbeResolveLabel(v, addrReset);
7701 resetAccumulator(pParse, pAggInfo);
7702 sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag);
7703 VdbeComment((v, "indicate accumulator empty"));
7704 sqlite3VdbeAddOp1(v, OP_Return, regReset);
7705
7706 if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){
7707 struct AggInfo_func *pF = &pAggInfo->aFunc[0];
7708 fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr);
7709 }
7710 } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */
7711 else {
7712 Table *pTab;
7713 if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){
7714 /* If isSimpleCount() returns a pointer to a Table structure, then
7715 ** the SQL statement is of the form:
7716 **
7717 ** SELECT count(*) FROM <tbl>
7718 **
7719 ** where the Table structure returned represents table <tbl>.
7720 **
7721 ** This statement is so common that it is optimized specially. The
7722 ** OP_Count instruction is executed either on the intkey table that
7723 ** contains the data for table <tbl> or on one of its indexes. It
7724 ** is better to execute the op on an index, as indexes are almost
7725 ** always spread across less pages than their corresponding tables.
7726 */
7727 const int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
7728 const int iCsr = pParse->nTab++; /* Cursor to scan b-tree */
7729 Index *pIdx; /* Iterator variable */
7730 KeyInfo *pKeyInfo = 0; /* Keyinfo for scanned index */
7731 Index *pBest = 0; /* Best index found so far */
7732 Pgno iRoot = pTab->tnum; /* Root page of scanned b-tree */
7733
7734 sqlite3CodeVerifySchema(pParse, iDb);
7735 sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
7736
7737 /* Search for the index that has the lowest scan cost.
7738 **
7739 ** (2011-04-15) Do not do a full scan of an unordered index.
7740 **
7741 ** (2013-10-03) Do not count the entries in a partial index.
7742 **
7743 ** In practice the KeyInfo structure will not be used. It is only
7744 ** passed to keep OP_OpenRead happy.
7745 */
7746 if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab);
7747 if( !p->pSrc->a[0].fg.notIndexed ){
7748 for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
7749 if( pIdx->bUnordered==0
7750 && pIdx->szIdxRow<pTab->szTabRow
7751 && pIdx->pPartIdxWhere==0
7752 && (!pBest || pIdx->szIdxRow<pBest->szIdxRow)
7753 ){
7754 pBest = pIdx;
7755 }
7756 }
7757 }
7758 if( pBest ){
7759 iRoot = pBest->tnum;
7760 pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pBest);
7761 }
7762
7763 /* Open a read-only cursor, execute the OP_Count, close the cursor. */
7764 sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, (int)iRoot, iDb, 1);
7765 if( pKeyInfo ){
7766 sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO);
7767 }
7768 sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem);
7769 sqlite3VdbeAddOp1(v, OP_Close, iCsr);
7770 explainSimpleCount(pParse, pTab, pBest);
7771 }else{
7772 int regAcc = 0; /* "populate accumulators" flag */
7773 ExprList *pDistinct = 0;
7774 u16 distFlag = 0;
7775 int eDist;
7776
7777 /* If there are accumulator registers but no min() or max() functions
7778 ** without FILTER clauses, allocate register regAcc. Register regAcc
7779 ** will contain 0 the first time the inner loop runs, and 1 thereafter.
7780 ** The code generated by updateAccumulator() uses this to ensure
7781 ** that the accumulator registers are (a) updated only once if
7782 ** there are no min() or max functions or (b) always updated for the
7783 ** first row visited by the aggregate, so that they are updated at
7784 ** least once even if the FILTER clause means the min() or max()
7785 ** function visits zero rows. */
7786 if( pAggInfo->nAccumulator ){
7787 for(i=0; i<pAggInfo->nFunc; i++){
7788 if( ExprHasProperty(pAggInfo->aFunc[i].pFExpr, EP_WinFunc) ){
7789 continue;
7790 }
7791 if( pAggInfo->aFunc[i].pFunc->funcFlags&SQLITE_FUNC_NEEDCOLL ){
7792 break;
7793 }
7794 }
7795 if( i==pAggInfo->nFunc ){
7796 regAcc = ++pParse->nMem;
7797 sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc);
7798 }
7799 }else if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 ){
7800 assert( ExprUseXList(pAggInfo->aFunc[0].pFExpr) );
7801 pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList;
7802 distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0;
7803 }
7804
7805 /* This case runs if the aggregate has no GROUP BY clause. The
7806 ** processing is much simpler since there is only a single row
7807 ** of output.
7808 */
7809 assert( p->pGroupBy==0 );
7810 resetAccumulator(pParse, pAggInfo);
7811
7812 /* If this query is a candidate for the min/max optimization, then
7813 ** minMaxFlag will have been previously set to either
7814 ** WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX and pMinMaxOrderBy will
7815 ** be an appropriate ORDER BY expression for the optimization.
7816 */
7817 assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 );
7818 assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 );
7819
7820 SELECTTRACE(1,pParse,p,("WhereBegin\n"));
7821 pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy,
7822 pDistinct, p, minMaxFlag|distFlag, 0);
7823 if( pWInfo==0 ){
7824 goto select_end;
7825 }
7826 SELECTTRACE(1,pParse,p,("WhereBegin returns\n"));
7827 eDist = sqlite3WhereIsDistinct(pWInfo);
7828 updateAccumulator(pParse, regAcc, pAggInfo, eDist);
7829 if( eDist!=WHERE_DISTINCT_NOOP ){
7830 struct AggInfo_func *pF = pAggInfo->aFunc;
7831 if( pF ){
7832 fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr);
7833 }
7834 }
7835
7836 if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc);
7837 if( minMaxFlag ){
7838 sqlite3WhereMinMaxOptEarlyOut(v, pWInfo);
7839 }
7840 SELECTTRACE(1,pParse,p,("WhereEnd\n"));
7841 sqlite3WhereEnd(pWInfo);
7842 finalizeAggFunctions(pParse, pAggInfo);
7843 }
7844
7845 sSort.pOrderBy = 0;
7846 sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL);
7847 selectInnerLoop(pParse, p, -1, 0, 0,
7848 pDest, addrEnd, addrEnd);
7849 }
7850 sqlite3VdbeResolveLabel(v, addrEnd);
7851
7852 } /* endif aggregate query */
7853
7854 if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){
7855 explainTempTable(pParse, "DISTINCT");
7856 }
7857
7858 /* If there is an ORDER BY clause, then we need to sort the results
7859 ** and send them to the callback one by one.
7860 */
7861 if( sSort.pOrderBy ){
7862 explainTempTable(pParse,
7863 sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY");
7864 assert( p->pEList==pEList );
7865 generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest);
7866 }
7867
7868 /* Jump here to skip this query
7869 */
7870 sqlite3VdbeResolveLabel(v, iEnd);
7871
7872 /* The SELECT has been coded. If there is an error in the Parse structure,
7873 ** set the return code to 1. Otherwise 0. */
7874 rc = (pParse->nErr>0);
7875
7876 /* Control jumps to here if an error is encountered above, or upon
7877 ** successful coding of the SELECT.
7878 */
7879select_end:
7880 assert( db->mallocFailed==0 || db->mallocFailed==1 );
7881 assert( db->mallocFailed==0 || pParse->nErr!=0 );
7882 sqlite3ExprListDelete(db, pMinMaxOrderBy);
7883#ifdef SQLITE_DEBUG
7884 if( pAggInfo && !db->mallocFailed ){
7885 for(i=0; i<pAggInfo->nColumn; i++){
7886 Expr *pExpr = pAggInfo->aCol[i].pCExpr;
7887 assert( pExpr!=0 );
7888 assert( pExpr->pAggInfo==pAggInfo );
7889 assert( pExpr->iAgg==i );
7890 }
7891 for(i=0; i<pAggInfo->nFunc; i++){
7892 Expr *pExpr = pAggInfo->aFunc[i].pFExpr;
7893 assert( pExpr!=0 );
7894 assert( pExpr->pAggInfo==pAggInfo );
7895 assert( pExpr->iAgg==i );
7896 }
7897 }
7898#endif
7899
7900#if TREETRACE_ENABLED
7901 SELECTTRACE(0x1,pParse,p,("end processing\n"));
7902 if( (sqlite3TreeTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){
7903 sqlite3TreeViewSelect(0, p, 0);
7904 }
7905#endif
7906 ExplainQueryPlanPop(pParse);
7907 return rc;
7908}
7909