1/*
2 * Legal Notice
3 *
4 * This document and associated source code (the "Work") is a part of a
5 * benchmark specification maintained by the TPC.
6 *
7 * The TPC reserves all right, title, and interest to the Work as provided
8 * under U.S. and international laws, including without limitation all patent
9 * and trademark rights therein.
10 *
11 * No Warranty
12 *
13 * 1.1 TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THE INFORMATION
14 * CONTAINED HEREIN IS PROVIDED "AS IS" AND WITH ALL FAULTS, AND THE
15 * AUTHORS AND DEVELOPERS OF THE WORK HEREBY DISCLAIM ALL OTHER
16 * WARRANTIES AND CONDITIONS, EITHER EXPRESS, IMPLIED OR STATUTORY,
17 * INCLUDING, BUT NOT LIMITED TO, ANY (IF ANY) IMPLIED WARRANTIES,
18 * DUTIES OR CONDITIONS OF MERCHANTABILITY, OF FITNESS FOR A PARTICULAR
19 * PURPOSE, OF ACCURACY OR COMPLETENESS OF RESPONSES, OF RESULTS, OF
20 * WORKMANLIKE EFFORT, OF LACK OF VIRUSES, AND OF LACK OF NEGLIGENCE.
21 * ALSO, THERE IS NO WARRANTY OR CONDITION OF TITLE, QUIET ENJOYMENT,
22 * QUIET POSSESSION, CORRESPONDENCE TO DESCRIPTION OR NON-INFRINGEMENT
23 * WITH REGARD TO THE WORK.
24 * 1.2 IN NO EVENT WILL ANY AUTHOR OR DEVELOPER OF THE WORK BE LIABLE TO
25 * ANY OTHER PARTY FOR ANY DAMAGES, INCLUDING BUT NOT LIMITED TO THE
26 * COST OF PROCURING SUBSTITUTE GOODS OR SERVICES, LOST PROFITS, LOSS
27 * OF USE, LOSS OF DATA, OR ANY INCIDENTAL, CONSEQUENTIAL, DIRECT,
28 * INDIRECT, OR SPECIAL DAMAGES WHETHER UNDER CONTRACT, TORT, WARRANTY,
29 * OR OTHERWISE, ARISING IN ANY WAY OUT OF THIS OR ANY OTHER AGREEMENT
30 * RELATING TO THE WORK, WHETHER OR NOT SUCH AUTHOR OR DEVELOPER HAD
31 * ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES.
32 *
33 * Contributors
34 * - Sergey Vasilevskiy
35 * - Doug Johnson
36 */
37
38/*
39 * This file contains a class that acts as a client to the table
40 * generation classes (EGenTables) and to the loader classes (EGenBaseLoader).
41 * It provides routines for generating and loading the table data or its
42 * subset.
43 */
44
45#include "main/EGenGenerateAndLoad_stdafx.h"
46#include "main/DriverParamSettings.h"
47
48#include "main/TableTypes.h"
49#include "main/ChargeTable.h"
50#include "main/CommissionRateTable.h"
51#include "main/ExchangeTable.h"
52#include "main/IndustryTable.h"
53#include "main/SectorTable.h"
54#include "main/StatusTypeTable.h"
55#include "main/TaxRateTable.h"
56#include "main/TradeTypeTable.h"
57#include "main/ZipCodeTable.h"
58
59using namespace TPCE;
60
61/*
62 * Constructor.
63 *
64 * PARAMETERS:
65 * IN dfm - in-memory representation of input flat
66 * files IN inputFiles - in-memory representation of input flat files
67 * IN iCustomerCount - number of customers to build (for this
68 * class instance) IN iStartFromCustomer - first customer id IN
69 * iTotalCustomers - total number of customers in the database IN
70 * iLoadUnitSize - minimal number of customers that can be build (should
71 * always be 1000) IN iScaleFactor - number of customers for 1tpsE IN
72 * iDaysOfInitialTrades- number of 8-hour days of initial trades per customer IN
73 * pLoaderFactory - factory to create loader classes IN pLogger -
74 * parameter logging interface IN pOutput - interface to output
75 * information to a user during the build process IN szInDir -
76 * input flat file directory needed for tables loaded from flat files
77 *
78 * RETURNS:
79 * not applicable.
80 */
81CGenerateAndLoad::CGenerateAndLoad(const DataFileManager &dfm, TIdent iCustomerCount, TIdent iStartFromCustomer,
82 TIdent iTotalCustomers, UINT iLoadUnitSize, UINT iScaleFactor,
83 UINT iDaysOfInitialTrades, CBaseLoaderFactory *pLoaderFactory, CBaseLogger *pLogger,
84 CGenerateAndLoadBaseOutput *pOutput, bool bCacheEnabled)
85 : m_dfm(dfm), m_iStartFromCustomer(iStartFromCustomer), m_iCustomerCount(iCustomerCount),
86 m_iTotalCustomers(iTotalCustomers), m_iLoadUnitSize(iLoadUnitSize), m_iScaleFactor(iScaleFactor),
87 m_iHoursOfInitialTrades(iDaysOfInitialTrades * HoursPerWorkDay), m_pLoaderFactory(pLoaderFactory),
88 m_pOutput(pOutput), m_pLogger(pLogger), m_LoaderSettings(iTotalCustomers, iTotalCustomers, iStartFromCustomer,
89 iCustomerCount, iScaleFactor, iDaysOfInitialTrades),
90 m_bCacheEnabled(bCacheEnabled) {
91 // Copy input flat file directory needed for tables loaded from flat files.
92
93 // Log Parameters
94 m_pLogger->SendToLogger(m_LoaderSettings);
95}
96
97/*
98 * Generate and load ADDRESS table.
99 *
100 * PARAMETERS:
101 * none.
102 *
103 * RETURNS:
104 * none.
105 */
106void CGenerateAndLoad::GenerateAndLoadAddress() {
107 bool bRet;
108 CAddressTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer,
109 // do not generate exchange and company addresses
110 // if the starting customer is not 1
111 m_iStartFromCustomer != iDefaultStartFromCustomer);
112 CBaseLoader<ADDRESS_ROW> *pLoad = m_pLoaderFactory->CreateAddressLoader();
113 INT64 iCnt = 0;
114
115 m_pOutput->OutputStart("Generating ADDRESS table...");
116
117 pLoad->Init();
118
119 do {
120 bRet = Table.GenerateNextRecord();
121
122 pLoad->WriteNextRecord(Table.GetRow());
123
124 if (++iCnt % 20000 == 0) { // output progress
125 m_pOutput->OutputProgress(".");
126 }
127
128 } while (bRet);
129
130 pLoad->FinishLoad(); // commit
131 delete pLoad;
132
133 m_pOutput->OutputComplete("loaded.");
134 m_pOutput->OutputNewline();
135 m_pOutput->OutputNewline();
136}
137
138/*
139 * Generate and load CHARGE table.
140 *
141 * PARAMETERS:
142 * none.
143 *
144 * RETURNS:
145 * none.
146 */
147void CGenerateAndLoad::GenerateAndLoadCharge() {
148 CChargeTable Table(m_dfm.ChargeDataFile());
149 CBaseLoader<CHARGE_ROW> *pLoad = m_pLoaderFactory->CreateChargeLoader();
150
151 m_pOutput->OutputStart("Generating CHARGE table...");
152
153 pLoad->Init();
154
155 while (Table.GenerateNextRecord()) {
156 pLoad->WriteNextRecord(Table.GetRow());
157 }
158 pLoad->FinishLoad(); // commit
159 delete pLoad;
160
161 m_pOutput->OutputComplete("loaded.");
162 m_pOutput->OutputNewline();
163 m_pOutput->OutputNewline();
164}
165
166/*
167 * Generate and load COMMISSION_RATE table.
168 *
169 * PARAMETERS:
170 * none.
171 *
172 * RETURNS:
173 * none.
174 */
175void CGenerateAndLoad::GenerateAndLoadCommissionRate() {
176 CCommissionRateTable Table(m_dfm.CommissionRateDataFile());
177 CBaseLoader<COMMISSION_RATE_ROW> *pLoad = m_pLoaderFactory->CreateCommissionRateLoader();
178
179 m_pOutput->OutputStart("Generating COMMISSION_RATE table...");
180
181 pLoad->Init();
182
183 while (Table.GenerateNextRecord()) {
184 pLoad->WriteNextRecord(Table.GetRow());
185 }
186 pLoad->FinishLoad(); // commit
187 delete pLoad;
188
189 m_pOutput->OutputComplete("loaded.");
190 m_pOutput->OutputNewline();
191 m_pOutput->OutputNewline();
192}
193
194/*
195 * Generate and load COMPANY table.
196 *
197 * PARAMETERS:
198 * none.
199 *
200 * RETURNS:
201 * none.
202 */
203void CGenerateAndLoad::GenerateAndLoadCompany() {
204 bool bRet;
205 CCompanyTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer);
206 CBaseLoader<COMPANY_ROW> *pLoad = m_pLoaderFactory->CreateCompanyLoader();
207
208 m_pOutput->OutputStart("Generating COMPANY table...");
209
210 pLoad->Init();
211
212 do {
213 bRet = Table.GenerateNextRecord();
214
215 pLoad->WriteNextRecord(Table.GetRow());
216
217 } while (bRet);
218
219 pLoad->FinishLoad(); // commit
220
221 delete pLoad;
222
223 m_pOutput->OutputComplete("loaded.");
224 m_pOutput->OutputNewline();
225 m_pOutput->OutputNewline();
226}
227
228/*
229 * Generate and load COMPANY_COMPETITOR table.
230 *
231 * PARAMETERS:
232 * none.
233 *
234 * RETURNS:
235 * none.
236 */
237void CGenerateAndLoad::GenerateAndLoadCompanyCompetitor() {
238 bool bRet;
239 CCompanyCompetitorTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer);
240 CBaseLoader<COMPANY_COMPETITOR_ROW> *pLoad = m_pLoaderFactory->CreateCompanyCompetitorLoader();
241
242 m_pOutput->OutputStart("Generating COMPANY_COMPETITOR table...");
243
244 pLoad->Init();
245
246 do {
247 bRet = Table.GenerateNextRecord();
248
249 pLoad->WriteNextRecord(Table.GetRow());
250
251 } while (bRet);
252
253 pLoad->FinishLoad(); // commit
254
255 delete pLoad;
256
257 m_pOutput->OutputComplete("loaded.");
258 m_pOutput->OutputNewline();
259 m_pOutput->OutputNewline();
260}
261
262/*
263 * Generate and load CUSTOMER table.
264 *
265 * PARAMETERS:
266 * none.
267 *
268 * RETURNS:
269 * none.
270 */
271void CGenerateAndLoad::GenerateAndLoadCustomer() {
272 bool bRet;
273 CCustomerTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer);
274 CBaseLoader<CUSTOMER_ROW> *pLoad = m_pLoaderFactory->CreateCustomerLoader();
275 INT64 iCnt = 0;
276
277 m_pOutput->OutputStart("Generating CUSTOMER table...");
278
279 pLoad->Init();
280
281 do {
282 bRet = Table.GenerateNextRecord();
283
284 pLoad->WriteNextRecord(Table.GetRow());
285
286 if (++iCnt % 20000 == 0) {
287 m_pOutput->OutputProgress("."); // output progress
288 }
289
290 } while (bRet);
291
292 pLoad->FinishLoad(); // commit
293
294 delete pLoad;
295
296 m_pOutput->OutputComplete("loaded.");
297 m_pOutput->OutputNewline();
298 m_pOutput->OutputNewline();
299}
300
301/*
302 * Generate and load CUSTOMER_ACCOUNT, ACCOUNT_PERMISSION table.
303 *
304 * PARAMETERS:
305 * none.
306 *
307 * RETURNS:
308 * none.
309 */
310void CGenerateAndLoad::GenerateAndLoadCustomerAccountAndAccountPermission() {
311 bool bRet;
312 CCustomerAccountsAndPermissionsTable Table(m_dfm, m_iLoadUnitSize, m_iCustomerCount, m_iStartFromCustomer);
313 CBaseLoader<CUSTOMER_ACCOUNT_ROW> *pCALoad = m_pLoaderFactory->CreateCustomerAccountLoader();
314 CBaseLoader<ACCOUNT_PERMISSION_ROW> *pAPLoad = m_pLoaderFactory->CreateAccountPermissionLoader();
315 INT64 iCnt = 0;
316 UINT i;
317
318 m_pOutput->OutputStart("Generating CUSTOMER_ACCOUNT table and ACCOUNT_PERMISSION table...");
319
320 pCALoad->Init();
321 pAPLoad->Init();
322
323 do {
324 bRet = Table.GenerateNextRecord();
325
326 pCALoad->WriteNextRecord(Table.GetCARow());
327
328 for (i = 0; i < Table.GetCAPermsCount(); ++i) {
329
330 pAPLoad->WriteNextRecord(Table.GetAPRow(i));
331 }
332
333 if (++iCnt % 10000 == 0) {
334 m_pOutput->OutputProgress("."); // output progress
335 }
336
337 // Commit rows every so often
338 if (iCnt % 10000 == 0) {
339 pCALoad->Commit();
340 pAPLoad->Commit();
341 }
342
343 } while (bRet);
344 pCALoad->FinishLoad(); // commit
345 pAPLoad->FinishLoad(); // commit
346 delete pCALoad;
347 delete pAPLoad;
348
349 m_pOutput->OutputComplete("loaded.");
350 m_pOutput->OutputNewline();
351 m_pOutput->OutputNewline();
352}
353
354/*
355 * Generate and load CUSTOMER_TAXRATE table.
356 *
357 * PARAMETERS:
358 * none.
359 *
360 * RETURNS:
361 * none.
362 */
363void CGenerateAndLoad::GenerateAndLoadCustomerTaxrate() {
364 bool bRet;
365 CCustomerTaxRateTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer);
366 CBaseLoader<CUSTOMER_TAXRATE_ROW> *pLoad = m_pLoaderFactory->CreateCustomerTaxrateLoader();
367 INT64 iCnt = 0;
368 UINT i;
369
370 m_pOutput->OutputStart("Generating CUSTOMER_TAX_RATE table...");
371
372 pLoad->Init();
373
374 do {
375 bRet = Table.GenerateNextRecord();
376
377 for (i = 0; i < Table.GetTaxRatesCount(); ++i) {
378 pLoad->WriteNextRecord(Table.GetRowByIndex(i));
379
380 if (++iCnt % 20000 == 0) {
381 m_pOutput->OutputProgress("."); // output progress
382 }
383 }
384 } while (bRet);
385
386 pLoad->FinishLoad(); // commit
387
388 delete pLoad;
389
390 m_pOutput->OutputComplete("loaded.");
391 m_pOutput->OutputNewline();
392 m_pOutput->OutputNewline();
393}
394
395/*
396 * Generate and load DAILY_MARKET table.
397 *
398 * PARAMETERS:
399 * none.
400 *
401 * RETURNS:
402 * none.
403 */
404void CGenerateAndLoad::GenerateAndLoadDailyMarket() {
405 bool bRet;
406 CDailyMarketTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer);
407 CBaseLoader<DAILY_MARKET_ROW> *pLoad = m_pLoaderFactory->CreateDailyMarketLoader();
408 INT64 iCnt = 0;
409
410 m_pOutput->OutputStart("Generating DAILY_MARKET table...");
411
412 pLoad->Init();
413
414 do {
415 bRet = Table.GenerateNextRecord();
416
417 pLoad->WriteNextRecord(Table.GetRow());
418
419 if (++iCnt % 20000 == 0) {
420 m_pOutput->OutputProgress("."); // output progress
421 }
422
423 if (iCnt % 6525 == 0) {
424 pLoad->Commit(); // commit rows every 5 securities (1305
425 // rows/security)
426 }
427
428 } while (bRet);
429
430 pLoad->FinishLoad(); // commit
431
432 delete pLoad;
433
434 m_pOutput->OutputComplete("loaded.");
435 m_pOutput->OutputNewline();
436 m_pOutput->OutputNewline();
437}
438
439/*
440 * Generate and load EXCHANGE table.
441 *
442 * PARAMETERS:
443 * none.
444 *
445 * RETURNS:
446 * none.
447 */
448void CGenerateAndLoad::GenerateAndLoadExchange() {
449 CExchangeTable Table(m_dfm.ExchangeDataFile(), m_iCustomerCount);
450 CBaseLoader<EXCHANGE_ROW> *pLoad = m_pLoaderFactory->CreateExchangeLoader();
451
452 m_pOutput->OutputStart("Generating EXCHANGE table...");
453
454 pLoad->Init();
455
456 while (Table.GenerateNextRecord()) {
457 pLoad->WriteNextRecord(Table.GetRow());
458 }
459 pLoad->FinishLoad(); // commit
460 delete pLoad;
461
462 m_pOutput->OutputComplete("loaded.");
463 m_pOutput->OutputNewline();
464 m_pOutput->OutputNewline();
465}
466
467/*
468 * Generate and load FINANCIAL table.
469 *
470 * PARAMETERS:
471 * none.
472 *
473 * RETURNS:
474 * none.
475 */
476void CGenerateAndLoad::GenerateAndLoadFinancial() {
477 bool bRet;
478 CFinancialTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer);
479 CBaseLoader<FINANCIAL_ROW> *pLoad = m_pLoaderFactory->CreateFinancialLoader();
480 INT64 iCnt = 0;
481
482 m_pOutput->OutputStart("Generating FINANCIAL table...");
483
484 pLoad->Init();
485
486 do {
487 bRet = Table.GenerateNextRecord();
488
489 pLoad->WriteNextRecord(Table.GetRow());
490
491 if (++iCnt % 20000 == 0) {
492 m_pOutput->OutputProgress("."); // output progress
493 }
494
495 if (iCnt % 5000 == 0) {
496 pLoad->Commit(); // commit rows every 250 companies (20 rows/company)
497 }
498
499 } while (bRet);
500
501 pLoad->FinishLoad(); // commit
502
503 delete pLoad;
504
505 m_pOutput->OutputComplete("loaded.");
506 m_pOutput->OutputNewline();
507 m_pOutput->OutputNewline();
508}
509
510/*
511 * Generate and load HOLDING, HOLDING_HISTORY, TRADE, TRADE_HISTORY,
512 * SETTLEMENT, CASH_TRANSACTION, BROKER table.
513 *
514 * PARAMETERS:
515 * none.
516 *
517 * RETURNS:
518 * none.
519 */
520void CGenerateAndLoad::GenerateAndLoadHoldingAndTrade() {
521 bool bRet;
522 CTradeGen *pTradeGen;
523
524 CBaseLoader<HOLDING_ROW> *pHoldingsLoad;
525 CBaseLoader<HOLDING_HISTORY_ROW> *pHoldingHistoryLoad;
526 CBaseLoader<HOLDING_SUMMARY_ROW> *pHoldingSummaryLoad;
527 CBaseLoader<TRADE_ROW> *pTradesLoad;
528 // Not loading TRADE_REQUEST table (it'll be quickly populated during a run)
529 // CBaseLoader<TRADE_REQUEST_ROW>* pRequestsLoad;
530 CBaseLoader<SETTLEMENT_ROW> *pSettlementLoad;
531 CBaseLoader<TRADE_HISTORY_ROW> *pHistoryLoad;
532 CBaseLoader<CASH_TRANSACTION_ROW> *pCashLoad;
533 CBaseLoader<BROKER_ROW> *pBrokerLoad;
534 int iCnt = 0;
535 int i;
536 int iCurrentLoadUnit = 1;
537 char szCurrentLoadUnit[11];
538
539 pHoldingsLoad = m_pLoaderFactory->CreateHoldingLoader();
540 pHoldingHistoryLoad = m_pLoaderFactory->CreateHoldingHistoryLoader();
541 pHoldingSummaryLoad = m_pLoaderFactory->CreateHoldingSummaryLoader();
542 pTradesLoad = m_pLoaderFactory->CreateTradeLoader();
543 // Not loading TRADE_REQUEST table (it'll be quickly populated during a run)
544 // pRequestsLoad = m_pLoaderFactory->CreateTradeRequestLoader();
545 pSettlementLoad = m_pLoaderFactory->CreateSettlementLoader();
546 pHistoryLoad = m_pLoaderFactory->CreateTradeHistoryLoader();
547 pCashLoad = m_pLoaderFactory->CreateCashTransactionLoader();
548 pBrokerLoad = m_pLoaderFactory->CreateBrokerLoader();
549
550 m_pOutput->OutputStart("Generating TRADE, SETTLEMENT, TRADE HISTORY, CASH TRANSACTION, "
551 "HOLDING_HISTORY, HOLDING_SUMMARY, HOLDING, and BROKER tables...");
552
553 pTradeGen = new CTradeGen(m_dfm, m_iCustomerCount, m_iStartFromCustomer, m_iTotalCustomers, m_iLoadUnitSize,
554 m_iScaleFactor, m_iHoursOfInitialTrades, m_bCacheEnabled);
555
556 // Generate and load one load unit at a time.
557 //
558 do {
559 pTradesLoad->Init();
560 pSettlementLoad->Init();
561 pHistoryLoad->Init();
562 pCashLoad->Init();
563 pBrokerLoad->Init();
564 pHoldingHistoryLoad->Init();
565 pHoldingsLoad->Init();
566 pHoldingSummaryLoad->Init();
567 // Not loading TRADE_REQUEST table
568 // pRequestsLoad->Init();
569
570 // Generate and load trades for this load unit.
571 //
572 do {
573 bRet = pTradeGen->GenerateNextTrade();
574
575 pTradesLoad->WriteNextRecord(pTradeGen->GetTradeRow());
576
577 for (i = 0; i < pTradeGen->GetTradeHistoryRowCount(); ++i) {
578 pHistoryLoad->WriteNextRecord(pTradeGen->GetTradeHistoryRow(i));
579 }
580
581 if (pTradeGen->GetSettlementRowCount()) {
582 pSettlementLoad->WriteNextRecord(pTradeGen->GetSettlementRow());
583 }
584
585 if (pTradeGen->GetCashTransactionRowCount()) {
586 pCashLoad->WriteNextRecord(pTradeGen->GetCashTransactionRow());
587 }
588
589 for (i = 0; i < pTradeGen->GetHoldingHistoryRowCount(); ++i) {
590 pHoldingHistoryLoad->WriteNextRecord(pTradeGen->GetHoldingHistoryRow(i));
591 }
592
593 /*if ((pTradeGen->GetTradeRow())->m_iTradeStatus == eCompleted) //
594 Not loading TRADE_REQUEST table
595 {
596 pRequestsLoad->WriteNextRecord(pTradeGen->GetTradeRequestRow());
597 }*/
598
599 if (++iCnt % 10000 == 0) {
600 m_pOutput->OutputProgress("."); // output progress
601 }
602
603 // Commit rows every so often
604 if (iCnt % 10000 == 0) {
605 pTradesLoad->Commit(); // commit
606 pSettlementLoad->Commit(); // commit
607 pHistoryLoad->Commit(); // commit
608 pCashLoad->Commit();
609 pHoldingHistoryLoad->Commit(); // commit
610 // Not loading TRADE_REQUEST table
611 // pRequestsLoad->Commit(); //commit
612 }
613
614 } while (bRet);
615
616 // After trades generate and load BROKER table.
617 //
618 do {
619 bRet = pTradeGen->GenerateNextBrokerRecord();
620
621 pBrokerLoad->WriteNextRecord(pTradeGen->GetBrokerRow());
622
623 // Commit rows every so often
624 if (++iCnt % 10000 == 0) {
625 pBrokerLoad->Commit(); // commit
626 }
627 } while (bRet);
628
629 m_pOutput->OutputProgress("t");
630
631 // Now generate and load HOLDING_SUMMARY rows for this load unit.
632 //
633 do {
634 bRet = pTradeGen->GenerateNextHoldingSummaryRow();
635
636 pHoldingSummaryLoad->WriteNextRecord(pTradeGen->GetHoldingSummaryRow());
637
638 if (++iCnt % 10000 == 0) {
639 m_pOutput->OutputProgress("."); // output progress
640 }
641
642 // Commit rows every so often
643 if (iCnt % 10000 == 0) {
644 pHoldingSummaryLoad->Commit(); // commit
645 }
646 } while (bRet);
647
648 // Now generate and load holdings for this load unit.
649 //
650 do {
651 bRet = pTradeGen->GenerateNextHolding();
652
653 pHoldingsLoad->WriteNextRecord(pTradeGen->GetHoldingRow());
654
655 if (++iCnt % 10000 == 0) {
656 m_pOutput->OutputProgress("."); // output progress
657 }
658
659 // Commit rows every so often
660 if (iCnt % 10000 == 0) {
661 pHoldingsLoad->Commit(); // commit
662 }
663 } while (bRet);
664
665 pTradesLoad->FinishLoad(); // commit
666 pSettlementLoad->FinishLoad(); // commit
667 pHistoryLoad->FinishLoad(); // commit
668 pCashLoad->FinishLoad(); // commit
669 pBrokerLoad->FinishLoad(); // commit
670 pHoldingHistoryLoad->FinishLoad(); // commit
671 pHoldingsLoad->FinishLoad(); // commit
672 pHoldingSummaryLoad->FinishLoad(); // commit
673 // Not loading TRADE_REQUEST table
674 // pRequestsLoad->FinishLoad(); //commit
675
676 // Output unit number for information
677 snprintf(szCurrentLoadUnit, sizeof(szCurrentLoadUnit), "%d", iCurrentLoadUnit++);
678
679 m_pOutput->OutputProgress(szCurrentLoadUnit);
680
681 } while (pTradeGen->InitNextLoadUnit());
682
683 delete pHoldingsLoad;
684 delete pHoldingHistoryLoad;
685 delete pHoldingSummaryLoad;
686 delete pTradesLoad;
687 // Not loading TRADE_REQUEST table
688 // delete pRequestsLoad;
689 delete pSettlementLoad;
690 delete pHistoryLoad;
691 delete pCashLoad;
692 delete pBrokerLoad;
693
694 delete pTradeGen;
695
696 m_pOutput->OutputComplete(".loaded.");
697 m_pOutput->OutputNewline();
698 m_pOutput->OutputNewline();
699}
700
701/*
702 * Generate and load INDUSTRY table.
703 *
704 * PARAMETERS:
705 * none.
706 *
707 * RETURNS:
708 * none.
709 */
710void CGenerateAndLoad::GenerateAndLoadIndustry() {
711 CIndustryTable Table(m_dfm.IndustryDataFile());
712 CBaseLoader<INDUSTRY_ROW> *pLoad = m_pLoaderFactory->CreateIndustryLoader();
713
714 m_pOutput->OutputStart("Generating INDUSTRY table...");
715
716 pLoad->Init();
717
718 while (Table.GenerateNextRecord()) {
719 pLoad->WriteNextRecord(Table.GetRow());
720 }
721 pLoad->FinishLoad(); // commit
722 delete pLoad;
723
724 m_pOutput->OutputComplete("loaded.");
725 m_pOutput->OutputNewline();
726 m_pOutput->OutputNewline();
727}
728
729/*
730 * Generate and load LAST_TRADE table.
731 *
732 * PARAMETERS:
733 * none.
734 *
735 * RETURNS:
736 * none.
737 */
738void CGenerateAndLoad::GenerateAndLoadLastTrade() {
739 bool bRet;
740 CLastTradeTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer, m_iHoursOfInitialTrades);
741 CBaseLoader<LAST_TRADE_ROW> *pLoad = m_pLoaderFactory->CreateLastTradeLoader();
742
743 m_pOutput->OutputStart("Generating LAST TRADE table...");
744
745 pLoad->Init();
746
747 do {
748 bRet = Table.GenerateNextRecord();
749
750 pLoad->WriteNextRecord(Table.GetRow());
751
752 } while (bRet);
753
754 pLoad->FinishLoad(); // commit
755
756 delete pLoad;
757
758 m_pOutput->OutputComplete("loaded.");
759 m_pOutput->OutputNewline();
760 m_pOutput->OutputNewline();
761}
762
763/*
764 * Generate and load NEWS_ITEM, NEWS_XREF table.
765 *
766 * PARAMETERS:
767 * none.
768 *
769 * RETURNS:
770 * none.
771 */
772void CGenerateAndLoad::GenerateAndLoadNewsItemAndNewsXRef() {
773 bool bRet;
774 // allocated on the heap because contains 100KB item
775 CNewsItemAndXRefTable *pTable =
776 new CNewsItemAndXRefTable(m_dfm, m_iCustomerCount, m_iStartFromCustomer, m_iHoursOfInitialTrades);
777 CBaseLoader<NEWS_ITEM_ROW> *pNewsItemLoad = m_pLoaderFactory->CreateNewsItemLoader();
778 CBaseLoader<NEWS_XREF_ROW> *pNewsXRefLoad = m_pLoaderFactory->CreateNewsXRefLoader();
779 INT64 iCnt = 0;
780
781 m_pOutput->OutputStart("Generating NEWS_ITEM and NEWS_XREF table...");
782
783 pNewsItemLoad->Init();
784 pNewsXRefLoad->Init();
785
786 do {
787 bRet = pTable->GenerateNextRecord();
788
789 pNewsItemLoad->WriteNextRecord(pTable->GetNewsItemRow());
790
791 pNewsXRefLoad->WriteNextRecord(pTable->GetNewsXRefRow());
792
793 if (++iCnt % 1000 == 0) // output progress every 1000 rows because each
794 // row generation takes a lot of time
795 {
796 m_pOutput->OutputProgress("."); // output progress
797
798 pNewsItemLoad->Commit();
799 pNewsXRefLoad->Commit();
800 }
801
802 } while (bRet);
803 pNewsItemLoad->FinishLoad(); // commit
804 pNewsXRefLoad->FinishLoad(); // commit
805 delete pNewsItemLoad;
806 delete pNewsXRefLoad;
807 delete pTable;
808
809 m_pOutput->OutputComplete("loaded.");
810 m_pOutput->OutputNewline();
811 m_pOutput->OutputNewline();
812}
813
814/*
815 * Generate and load SECTOR table.
816 *
817 * PARAMETERS:
818 * none.
819 *
820 * RETURNS:
821 * none.
822 */
823void CGenerateAndLoad::GenerateAndLoadSector() {
824 CSectorTable Table(m_dfm.SectorDataFile());
825 CBaseLoader<SECTOR_ROW> *pLoad = m_pLoaderFactory->CreateSectorLoader();
826
827 m_pOutput->OutputStart("Generating SECTOR table...");
828
829 pLoad->Init();
830
831 while (Table.GenerateNextRecord()) {
832 pLoad->WriteNextRecord(Table.GetRow());
833 }
834 pLoad->FinishLoad(); // commit
835 delete pLoad;
836
837 m_pOutput->OutputComplete("loaded.");
838 m_pOutput->OutputNewline();
839 m_pOutput->OutputNewline();
840}
841
842/*
843 * Generate and load SECURITY table.
844 *
845 * PARAMETERS:
846 * none.
847 *
848 * RETURNS:
849 * none.
850 */
851void CGenerateAndLoad::GenerateAndLoadSecurity() {
852 bool bRet;
853 CSecurityTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer);
854 CBaseLoader<SECURITY_ROW> *pLoad = m_pLoaderFactory->CreateSecurityLoader();
855 INT64 iCnt = 0;
856
857 m_pOutput->OutputStart("Generating SECURITY table...");
858
859 pLoad->Init();
860
861 do {
862 bRet = Table.GenerateNextRecord();
863
864 pLoad->WriteNextRecord(Table.GetRow());
865
866 if (++iCnt % 20000 == 0) {
867 m_pOutput->OutputProgress("."); // output progress
868 }
869
870 } while (bRet);
871
872 pLoad->FinishLoad(); // commit
873
874 delete pLoad;
875
876 m_pOutput->OutputComplete("loaded.");
877 m_pOutput->OutputNewline();
878 m_pOutput->OutputNewline();
879}
880
881/*
882 * Generate and load STATUS_TYPE table.
883 *
884 * PARAMETERS:
885 * none.
886 *
887 * RETURNS:
888 * none.
889 */
890void CGenerateAndLoad::GenerateAndLoadStatusType() {
891 CStatusTypeTable Table(m_dfm.StatusTypeDataFile());
892 CBaseLoader<STATUS_TYPE_ROW> *pLoad = m_pLoaderFactory->CreateStatusTypeLoader();
893
894 m_pOutput->OutputStart("Generating STATUS_TYPE table...");
895
896 pLoad->Init();
897
898 while (Table.GenerateNextRecord()) {
899 pLoad->WriteNextRecord(Table.GetRow());
900 }
901 pLoad->FinishLoad(); // commit
902 delete pLoad;
903
904 m_pOutput->OutputComplete("loaded.");
905 m_pOutput->OutputNewline();
906 m_pOutput->OutputNewline();
907}
908
909/*
910 * Generate and load TAXRATE table.
911 *
912 * PARAMETERS:
913 * none.
914 *
915 * RETURNS:
916 * none.
917 */
918void CGenerateAndLoad::GenerateAndLoadTaxrate() {
919 // TaxRateFile df(m_dfm.TaxRatesCountryDataFile(),
920 // m_dfm.TaxRatesDivisionDataFile()); TaxRateTable_t Table(df);
921 TaxRateTable Table(m_dfm.TaxRateFile());
922 CBaseLoader<TAX_RATE_ROW> *pLoad = m_pLoaderFactory->CreateTaxRateLoader();
923
924 m_pOutput->OutputStart("Generating TAXRATE table...");
925
926 pLoad->Init();
927
928 while (Table.GenerateNextRecord()) {
929 pLoad->WriteNextRecord(Table.GetRow());
930 }
931 pLoad->FinishLoad(); // commit
932 delete pLoad;
933
934 m_pOutput->OutputComplete("loaded.");
935 m_pOutput->OutputNewline();
936 m_pOutput->OutputNewline();
937}
938
939/*
940 * Generate and load TRADE_TYPE table.
941 *
942 * PARAMETERS:
943 * none.
944 *
945 * RETURNS:
946 * none.
947 */
948void CGenerateAndLoad::GenerateAndLoadTradeType() {
949 CTradeTypeTable Table(m_dfm.TradeTypeDataFile());
950 CBaseLoader<TRADE_TYPE_ROW> *pLoad = m_pLoaderFactory->CreateTradeTypeLoader();
951
952 m_pOutput->OutputStart("Generating TRADE_TYPE table...");
953
954 pLoad->Init();
955
956 while (Table.GenerateNextRecord()) {
957 pLoad->WriteNextRecord(Table.GetRow());
958 }
959 pLoad->FinishLoad(); // commit
960 delete pLoad;
961
962 m_pOutput->OutputComplete("loaded.");
963 m_pOutput->OutputNewline();
964 m_pOutput->OutputNewline();
965}
966
967/*
968 * Generate and load WATCH_LIST, WATCH_ITEM table.
969 *
970 * PARAMETERS:
971 * none.
972 *
973 * RETURNS:
974 * none.
975 */
976void CGenerateAndLoad::GenerateAndLoadWatchListAndWatchItem() {
977 bool bRet;
978 CWatchListsAndItemsTable Table(m_dfm, m_iCustomerCount, m_iStartFromCustomer);
979 CBaseLoader<WATCH_LIST_ROW> *pWatchListsLoad = m_pLoaderFactory->CreateWatchListLoader();
980 CBaseLoader<WATCH_ITEM_ROW> *pWatchItemsLoad = m_pLoaderFactory->CreateWatchItemLoader();
981 INT64 iCnt = 0;
982 UINT i;
983
984 m_pOutput->OutputStart("Generating WATCH_LIST table and WATCH_ITEM table...");
985
986 pWatchListsLoad->Init();
987 pWatchItemsLoad->Init();
988
989 do {
990 bRet = Table.GenerateNextRecord();
991
992 pWatchListsLoad->WriteNextRecord(Table.GetWLRow());
993
994 for (i = 0; i < Table.GetWICount(); ++i) {
995 pWatchItemsLoad->WriteNextRecord(Table.GetWIRow(i));
996
997 if (++iCnt % 20000 == 0) {
998 m_pOutput->OutputProgress("."); // output progress
999
1000 pWatchListsLoad->Commit(); // commit
1001 pWatchItemsLoad->Commit(); // commit
1002 }
1003 }
1004 } while (bRet);
1005
1006 pWatchListsLoad->FinishLoad(); // commit
1007 pWatchItemsLoad->FinishLoad(); // commit
1008
1009 delete pWatchListsLoad;
1010 delete pWatchItemsLoad;
1011
1012 m_pOutput->OutputComplete("loaded.");
1013 m_pOutput->OutputNewline();
1014 m_pOutput->OutputNewline();
1015}
1016
1017/*
1018 * Generate and load ZIP_CODE table.
1019 *
1020 * PARAMETERS:
1021 * none.
1022 *
1023 * RETURNS:
1024 * none.
1025 */
1026void CGenerateAndLoad::GenerateAndLoadZipCode() {
1027 CZipCodeTable Table(m_dfm.ZipCodeDataFile());
1028 CBaseLoader<ZIP_CODE_ROW> *pLoad = m_pLoaderFactory->CreateZipCodeLoader();
1029
1030 m_pOutput->OutputStart("Generating ZIP_CODE table...");
1031
1032 pLoad->Init();
1033
1034 while (Table.GenerateNextRecord()) {
1035 pLoad->WriteNextRecord(Table.GetRow());
1036 }
1037 pLoad->FinishLoad(); // commit
1038 delete pLoad;
1039
1040 m_pOutput->OutputComplete("loaded.");
1041 m_pOutput->OutputNewline();
1042 m_pOutput->OutputNewline();
1043}
1044
1045/*
1046 * Generate and load All tables that are constant in size.
1047 *
1048 * Spec definition: Fixed tables.
1049 *
1050 * PARAMETERS:
1051 * none.
1052 *
1053 * RETURNS:
1054 * none.
1055 */
1056void CGenerateAndLoad::GenerateAndLoadFixedTables() {
1057 GenerateAndLoadCharge();
1058 GenerateAndLoadCommissionRate();
1059 GenerateAndLoadExchange();
1060 GenerateAndLoadIndustry();
1061 GenerateAndLoadSector();
1062 GenerateAndLoadStatusType();
1063 GenerateAndLoadTaxrate();
1064 GenerateAndLoadTradeType();
1065 GenerateAndLoadZipCode();
1066}
1067
1068/*
1069 * Generate and load All tables (except BROKER) that scale with the size of
1070 * the CUSTOMER table, but do not grow in runtime.
1071 *
1072 * Spec definition: Scaling tables.
1073 *
1074 * PARAMETERS:
1075 * none.
1076 *
1077 * RETURNS:
1078 * none.
1079 */
1080void CGenerateAndLoad::GenerateAndLoadScalingTables() {
1081 // Customer-related tables
1082 //
1083 GenerateAndLoadAddress();
1084 GenerateAndLoadCustomer();
1085 GenerateAndLoadCustomerAccountAndAccountPermission();
1086 GenerateAndLoadCustomerTaxrate();
1087 GenerateAndLoadWatchListAndWatchItem();
1088
1089 // Now security/company related tables
1090 //
1091 GenerateAndLoadCompany();
1092 GenerateAndLoadCompanyCompetitor();
1093 GenerateAndLoadDailyMarket();
1094 GenerateAndLoadFinancial();
1095 GenerateAndLoadLastTrade();
1096 GenerateAndLoadNewsItemAndNewsXRef();
1097 GenerateAndLoadSecurity();
1098}
1099
1100/*
1101 * Generate and load All trade related tables and BROKER (included here to
1102 * facilitate generation of a consistent database).
1103 *
1104 * Spec definition: Growing tables.
1105 *
1106 * PARAMETERS:
1107 * none.
1108 *
1109 * RETURNS:
1110 * none.
1111 */
1112void CGenerateAndLoad::GenerateAndLoadGrowingTables() {
1113 GenerateAndLoadHoldingAndTrade();
1114}
1115