MySQL 5.6.14 Source Code Document
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
testSystemRestart.cpp
1 /*
2  Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
3 
4  This program is free software; you can redistribute it and/or modify
5  it under the terms of the GNU General Public License as published by
6  the Free Software Foundation; version 2 of the License.
7 
8  This program is distributed in the hope that it will be useful,
9  but WITHOUT ANY WARRANTY; without even the implied warranty of
10  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  GNU General Public License for more details.
12 
13  You should have received a copy of the GNU General Public License
14  along with this program; if not, write to the Free Software
15  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
16 */
17 
18 #include <NDBT.hpp>
19 #include <NDBT_Test.hpp>
20 #include <HugoTransactions.hpp>
21 #include <UtilTransactions.hpp>
22 #include <NdbRestarter.hpp>
23 #include <Vector.hpp>
24 #include <signaldata/DumpStateOrd.hpp>
25 #include <NdbBackup.hpp>
26 #include <Bitmask.hpp>
27 
28 int runLoadTable(NDBT_Context* ctx, NDBT_Step* step){
29 
30  int records = ctx->getNumRecords();
31  HugoTransactions hugoTrans(*ctx->getTab());
32  if (hugoTrans.loadTable(GETNDB(step), records) != 0){
33  return NDBT_FAILED;
34  }
35  return NDBT_OK;
36 }
37 
38 int
39 clearOldBackups(NDBT_Context* ctx, NDBT_Step* step)
40 {
41  NdbBackup backup(GETNDB(step)->getNodeId());
42  backup.clearOldBackups();
43  return NDBT_OK;
44 }
45 
46 #define CHECK(b) if (!(b)) { \
47  g_err << "ERR: "<< step->getName() \
48  << " failed on line " << __LINE__ << endl; \
49  result = NDBT_FAILED; \
50  continue; }
51 
52 int runSystemRestart1(NDBT_Context* ctx, NDBT_Step* step){
53  Ndb* pNdb = GETNDB(step);
54  int result = NDBT_OK;
55  int timeout = 300;
56  Uint32 loops = ctx->getNumLoops();
57  int records = ctx->getNumRecords();
58  int count;
59  NdbRestarter restarter;
60  Uint32 i = 1;
61 
62  UtilTransactions utilTrans(*ctx->getTab());
63  HugoTransactions hugoTrans(*ctx->getTab());
64  while(i<=loops && result != NDBT_FAILED){
65 
66  ndbout << "Loop " << i << "/"<< loops <<" started" << endl;
67  /*
68  1. Load data
69  2. Restart cluster and verify records
70  3. Update records
71  4. Restart cluster and verify records
72  5. Delete half of the records
73  6. Restart cluster and verify records
74  7. Delete all records
75  8. Restart cluster and verify records
76  9. Insert, update, delete records
77  10. Restart cluster and verify records
78  11. Insert, update, delete records
79  12. Restart cluster with error insert 5020 and verify records
80  */
81  ndbout << "Loading records..." << endl;
82  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
83 
84  ndbout << "Restarting cluster" << endl;
85  CHECK(restarter.restartAll() == 0);
86  CHECK(restarter.waitClusterStarted(timeout) == 0);
87  CHECK(pNdb->waitUntilReady(timeout) == 0);
88 
89  ndbout << "Verifying records..." << endl;
90  CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0);
91  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
92  CHECK(count == records);
93 
94  ndbout << "Updating records..." << endl;
95  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
96 
97  ndbout << "Restarting cluster..." << endl;
98  CHECK(restarter.restartAll() == 0);
99  CHECK(restarter.waitClusterStarted(timeout) == 0);
100  CHECK(pNdb->waitUntilReady(timeout) == 0);
101 
102  ndbout << "Verifying records..." << endl;
103  CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0);
104  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
105  CHECK(count == records);
106 
107  ndbout << "Deleting 50% of records..." << endl;
108  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
109 
110  ndbout << "Restarting cluster..." << endl;
111  CHECK(restarter.restartAll() == 0);
112  CHECK(restarter.waitClusterStarted(timeout) == 0);
113  CHECK(pNdb->waitUntilReady(timeout) == 0);
114 
115  ndbout << "Verifying records..." << endl;
116  CHECK(hugoTrans.scanReadRecords(pNdb, records/2, 0, 64) == 0);
117  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
118  CHECK(count == (records/2));
119 
120  ndbout << "Deleting all records..." << endl;
121  CHECK(utilTrans.clearTable(pNdb, records/2) == 0);
122 
123  ndbout << "Restarting cluster..." << endl;
124  CHECK(restarter.restartAll() == 0);
125  CHECK(restarter.waitClusterStarted(timeout) == 0);
126  CHECK(pNdb->waitUntilReady(timeout) == 0);
127 
128  ndbout << "Verifying records..." << endl;
129  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
130  CHECK(count == 0);
131 
132  ndbout << "Doing it all..." << endl;
133  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
134  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
135  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
136  CHECK(hugoTrans.scanUpdateRecords(pNdb, records/2) == 0);
137  CHECK(utilTrans.clearTable(pNdb, records) == 0);
138  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
139  CHECK(utilTrans.clearTable(pNdb, records) == 0);
140  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
141  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
142  CHECK(utilTrans.clearTable(pNdb, records) == 0);
143 
144  ndbout << "Restarting cluster..." << endl;
145  CHECK(restarter.restartAll() == 0);
146  CHECK(restarter.waitClusterStarted(timeout) == 0);
147  CHECK(pNdb->waitUntilReady(timeout) == 0);
148 
149  ndbout << "Verifying records..." << endl;
150  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
151  CHECK(count == 0);
152 
153  ndbout << "Doing it all..." << endl;
154  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
155  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
156  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
157  CHECK(hugoTrans.scanUpdateRecords(pNdb, records/2) == 0);
158  CHECK(utilTrans.clearTable(pNdb, records) == 0);
159  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
160  CHECK(utilTrans.clearTable(pNdb, records) == 0);
161 
162  ndbout << "Restarting cluster with error insert 5020..." << endl;
163  CHECK(restarter.restartAll(false, true) == 0);
164  CHECK(restarter.waitClusterNoStart(timeout) == 0);
165  CHECK(restarter.insertErrorInAllNodes(5020) == 0);
166  CHECK(restarter.startAll() == 0);
167  CHECK(restarter.waitClusterStarted(timeout) == 0);
168  CHECK(pNdb->waitUntilReady(timeout) == 0);
169 
170  i++;
171  }
172 
173  ndbout << "runSystemRestart1 finished" << endl;
174 
175  return result;
176 }
177 
178 int runSystemRestart2(NDBT_Context* ctx, NDBT_Step* step){
179  Ndb* pNdb = GETNDB(step);
180  int result = NDBT_OK;
182  int timeout = 120;
183  Uint32 loops = ctx->getNumLoops();
184  int records = ctx->getNumRecords();
185  int count;
186  NdbRestarter restarter;
187  Uint32 i = 1;
188 
189  UtilTransactions utilTrans(*ctx->getTab());
190  HugoTransactions hugoTrans(*ctx->getTab());
191  while(i<=loops && result != NDBT_FAILED && !ctx->isTestStopped()){
192 
193  ndbout << "Loop " << i << "/"<< loops <<" started" << endl;
194  /* Use error 7070 to set time between LCP to it's min value
195  1. Load data
196  2. Restart cluster and verify records
197  3. Update records
198  4. Restart cluster and verify records
199  5. Delete half of the records
200  6. Restart cluster and verify records
201  7. Delete all records
202  8. Restart cluster and verify records
203  9. Insert, update, delete records
204  10. Restart cluster and verify records
205  */
206  int val = DumpStateOrd::DihMinTimeBetweenLCP;
207  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
208 
209  ndbout << "Loading records..." << endl;
210  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
211 
212  ndbout << "Restarting cluster" << endl;
213  CHECK(restarter.restartAll() == 0);
214  CHECK(restarter.waitClusterStarted(timeout) == 0);
215  {
216  int val = DumpStateOrd::DihMinTimeBetweenLCP;
217  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
218  }
219  CHECK(pNdb->waitUntilReady(timeout) == 0);
220 
221  ndbout << "Verifying records..." << endl;
222  CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0);
223  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
224  CHECK(count == records);
225 
226  ndbout << "Updating records..." << endl;
227  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
228 
229  ndbout << "Restarting cluster..." << endl;
230  CHECK(restarter.restartAll() == 0);
231  CHECK(restarter.waitClusterStarted(timeout) == 0);
232  {
233  int val = DumpStateOrd::DihMinTimeBetweenLCP;
234  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
235  }
236  CHECK(pNdb->waitUntilReady(timeout) == 0);
237 
238  ndbout << "Verifying records..." << endl;
239  CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0);
240  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
241  CHECK(count == records);
242 
243  ndbout << "Deleting 50% of records..." << endl;
244  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
245 
246  ndbout << "Restarting cluster..." << endl;
247  CHECK(restarter.restartAll() == 0);
248  CHECK(restarter.waitClusterStarted(timeout) == 0);
249  {
250  int val = DumpStateOrd::DihMinTimeBetweenLCP;
251  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
252  }
253  CHECK(pNdb->waitUntilReady(timeout) == 0);
254 
255  ndbout << "Verifying records..." << endl;
256  CHECK(hugoTrans.scanReadRecords(pNdb, records/2, 0, 64) == 0);
257  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
258  CHECK(count == (records/2));
259 
260  ndbout << "Deleting all records..." << endl;
261  CHECK(utilTrans.clearTable(pNdb, records/2) == 0);
262 
263  ndbout << "Restarting cluster..." << endl;
264  CHECK(restarter.restartAll() == 0);
265  CHECK(restarter.waitClusterStarted(timeout) == 0);
266  {
267  int val = DumpStateOrd::DihMinTimeBetweenLCP;
268  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
269  }
270  CHECK(pNdb->waitUntilReady(timeout) == 0);
271 
272  ndbout << "Verifying records..." << endl;
273  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
274  CHECK(count == 0);
275 
276  ndbout << "Doing it all..." << endl;
277  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
278  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
279  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
280  CHECK(hugoTrans.scanUpdateRecords(pNdb, records/2) == 0);
281  CHECK(utilTrans.clearTable(pNdb, records) == 0);
282  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
283  CHECK(utilTrans.clearTable(pNdb, records) == 0);
284  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
285  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
286  CHECK(utilTrans.clearTable(pNdb, records) == 0);
287 
288  ndbout << "Restarting cluster..." << endl;
289  CHECK(restarter.restartAll() == 0);
290  CHECK(restarter.waitClusterStarted(timeout) == 0);
291  {
292  int val = DumpStateOrd::DihMinTimeBetweenLCP;
293  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
294  }
295  CHECK(pNdb->waitUntilReady(timeout) == 0);
296 
297  ndbout << "Verifying records..." << endl;
298  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
299  CHECK(count == 0);
300 
301  i++;
302  }
303 
304  ndbout << "runSystemRestart2 finished" << endl;
305 
306  return result;
307 }
308 
309 int runSystemRestartTestUndoLog(NDBT_Context* ctx, NDBT_Step* step){
310  Ndb* pNdb = GETNDB(step);
311  int result = NDBT_OK;
312  int timeout = 300;
313  Uint32 loops = ctx->getNumLoops();
314  int records = ctx->getNumRecords();
315  int count;
316  NdbRestarter restarter;
317  Uint32 i = 1;
318 
319  int dump7080[2];
320  dump7080[0] = 7080;
321  dump7080[1] = ctx->getTab()->getTableId();
322 
323  UtilTransactions utilTrans(*ctx->getTab());
324  HugoTransactions hugoTrans(*ctx->getTab());
325  while(i<=loops && result != NDBT_FAILED){
326 
327  ndbout << "Loop " << i << "/"<< loops <<" started" << endl;
328  /*
329  1. Start LCP, turn on undologging but delay write of datapages.
330  2. Insert, update, delete records
331  3. Complete writing of data pages and finish LCP.
332  4. Restart cluster and verify records
333  */
334  // Use dump state 7080 to delay writing of datapages
335  // for the current table
336  ndbout << "Dump state: "<<dump7080[0]<<", "<<dump7080[1]<<endl;
337  CHECK(restarter.dumpStateAllNodes(dump7080, 2) == 0);
338  NdbSleep_SecSleep(10);
339 
340  ndbout << "Doing it all..." << endl;
341  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
342  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
343  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
344  CHECK(hugoTrans.scanUpdateRecords(pNdb, records/2) == 0);
345  CHECK(utilTrans.clearTable(pNdb, records) == 0);
346  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
347  CHECK(utilTrans.clearTable(pNdb, records) == 0);
348 
349  // Reset error and let LCP continue
350  CHECK(restarter.insertErrorInAllNodes(0) == 0);
351  NdbSleep_SecSleep(60);
352 
353  ndbout << "Restarting cluster..." << endl;
354  CHECK(restarter.restartAll() == 0);
355  CHECK(restarter.waitClusterStarted(timeout) == 0);
356  CHECK(pNdb->waitUntilReady(timeout) == 0);
357 
358  ndbout << "Verifying records..." << endl;
359  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
360  CHECK(count == 0);
361 
362  // Use dump state 7080 to delay writing of datapages
363  // for the current table
364  ndbout << "Dump state: "<<dump7080[0]<<", "<<dump7080[1]<<endl;
365  CHECK(restarter.dumpStateAllNodes(dump7080, 2) == 0);
366  NdbSleep_SecSleep(10);
367 
368  ndbout << "Doing it all, delete 50%..." << endl;
369  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
370  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
371  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
372 
373  // Reset error and let LCP continue
374  CHECK(restarter.insertErrorInAllNodes(0) == 0);
375  NdbSleep_SecSleep(20);
376 
377  ndbout << "Restarting cluster..." << endl;
378  CHECK(restarter.restartAll() == 0);
379  CHECK(restarter.waitClusterStarted(timeout) == 0);
380  CHECK(pNdb->waitUntilReady(timeout) == 0);
381 
382  ndbout << "Verifying records..." << endl;
383  CHECK(hugoTrans.scanReadRecords(pNdb, records/2, 0, 64) == 0);
384  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
385  CHECK(count == (records/2));
386  CHECK(utilTrans.clearTable(pNdb, records) == 0);
387 
388  i++;
389  }
390 
391  ndbout << "runSystemRestartTestUndoLog finished" << endl;
392 
393  return result;
394 }
395 
396 int runSystemRestartTestFullDb(NDBT_Context* ctx, NDBT_Step* step){
397  Ndb* pNdb = GETNDB(step);
398  int result = NDBT_OK;
399  int timeout = 300;
400  Uint32 loops = ctx->getNumLoops();
401  int count1, count2;
402  NdbRestarter restarter;
403  Uint32 i = 1;
404 
405  UtilTransactions utilTrans(*ctx->getTab());
406  HugoTransactions hugoTrans(*ctx->getTab());
407  while(i<=loops && result != NDBT_FAILED){
408 
409  ndbout << "Loop " << i << "/"<< loops <<" started" << endl;
410  /*
411  1. Load data until db reports it's full
412  2. Restart cluster and verify records
413  */
414  ndbout << "Filling up table..." << endl;
415  CHECK(hugoTrans.fillTable(pNdb) == 0);
416  CHECK(utilTrans.selectCount(pNdb, 64, &count1) == 0);
417  ndbout << "Db is full. Table has "<<count1 <<" records."<< endl;
418 
419  ndbout << "Restarting cluster" << endl;
420  CHECK(restarter.restartAll() == 0);
421  CHECK(restarter.waitClusterStarted(timeout) == 0);
422  CHECK(pNdb->waitUntilReady(timeout) == 0);
423 
424  ndbout << "Verifying records..." << endl;
425  CHECK(hugoTrans.scanReadRecords(pNdb, count1) == 0);
426  CHECK(utilTrans.selectCount(pNdb, 64, &count2) == 0);
427  CHECK(count1 == count2);
428 
429  ndbout << "Deleting all records..." << endl;
430  CHECK(utilTrans.clearTable2(pNdb, count1) == 0);
431 
432  ndbout << "Restarting cluster..." << endl;
433  CHECK(restarter.restartAll() == 0);
434  CHECK(restarter.waitClusterStarted(timeout) == 0);
435  CHECK(pNdb->waitUntilReady(timeout) == 0);
436 
437  ndbout << "Verifying records..." << endl;
438  CHECK(utilTrans.selectCount(pNdb, 64, &count1) == 0);
439  CHECK(count1 == 0);
440 
441  i++;
442  }
443 
444  ndbout << "runSystemRestartTestFullDb finished" << endl;
445 
446  return result;
447 }
448 
449 int runSystemRestart3(NDBT_Context* ctx, NDBT_Step* step){
450  Ndb* pNdb = GETNDB(step);
451  int result = NDBT_OK;
452  int timeout = 300;
453  Uint32 loops = ctx->getNumLoops();
454  int records = ctx->getNumRecords();
455  int count;
456  NdbRestarter restarter;
457  Uint32 i = 1;
458 
459  const Uint32 nodeCount = restarter.getNumDbNodes();
460  if(nodeCount < 2){
461  g_info << "SR3 - Needs atleast 2 nodes to test" << endl;
462  return NDBT_OK;
463  }
464 
465  Vector<int> nodeIds;
466  for(i = 0; i<nodeCount; i++)
467  nodeIds.push_back(restarter.getDbNodeId(i));
468 
469  Uint32 currentRestartNodeIndex = 0;
470  UtilTransactions utilTrans(*ctx->getTab());
471  HugoTransactions hugoTrans(*ctx->getTab());
472 
473  while(i<=loops && result != NDBT_FAILED){
474 
475  g_info << "Loop " << i << "/"<< loops <<" started" << endl;
488  g_info << "Loading records..." << endl;
489  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
490 
491  /*** 1 ***/
492  g_info << "1 - Stopping one node" << endl;
493  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
494  false,
495  true,
496  false) == 0);
497  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
498 
499  g_info << "Updating records..." << endl;
500  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
501 
502  g_info << "Restarting cluster..." << endl;
503  CHECK(restarter.restartAll() == 0);
504  CHECK(restarter.waitClusterStarted(timeout) == 0);
505  CHECK(pNdb->waitUntilReady(timeout) == 0);
506 
507  g_info << "Verifying records..." << endl;
508  CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0);
509  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
510  CHECK(count == records);
511 
512  g_info << "2 - Stopping one node" << endl;
513  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
514  false,
515  true,
516  false) == 0);
517  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
518 
519  g_info << "Deleting 50% of records..." << endl;
520  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
521 
522  g_info << "Restarting cluster..." << endl;
523  CHECK(restarter.restartAll() == 0);
524  CHECK(restarter.waitClusterStarted(timeout) == 0);
525  CHECK(pNdb->waitUntilReady(timeout) == 0);
526 
527  g_info << "Verifying records..." << endl;
528  CHECK(hugoTrans.scanReadRecords(pNdb, records/2, 0, 64) == 0);
529  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
530  CHECK(count == (records/2));
531 
532  g_info << "3 - Stopping one node" << endl;
533  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
534  false,
535  true,
536  false) == 0);
537  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
538  g_info << "Deleting all records..." << endl;
539  CHECK(utilTrans.clearTable(pNdb, records/2) == 0);
540 
541  g_info << "Restarting cluster..." << endl;
542  CHECK(restarter.restartAll() == 0);
543  CHECK(restarter.waitClusterStarted(timeout) == 0);
544  CHECK(pNdb->waitUntilReady(timeout) == 0);
545 
546  ndbout << "Verifying records..." << endl;
547  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
548  CHECK(count == 0);
549 
550  i++;
551  }
552 
553  g_info << "runSystemRestart3 finished" << endl;
554 
555  return result;
556 }
557 
558 int runSystemRestart4(NDBT_Context* ctx, NDBT_Step* step){
559  Ndb* pNdb = GETNDB(step);
560  int result = NDBT_OK;
561  int timeout = 300;
562  Uint32 loops = ctx->getNumLoops();
563  int records = ctx->getNumRecords();
564  int count;
565  NdbRestarter restarter;
566  Uint32 i = 1;
567 
568  const Uint32 nodeCount = restarter.getNumDbNodes();
569  if(nodeCount < 2){
570  g_info << "SR4 - Needs atleast 2 nodes to test" << endl;
571  return NDBT_OK;
572  }
573 
574  Vector<int> nodeIds;
575  for(i = 0; i<nodeCount; i++)
576  nodeIds.push_back(restarter.getDbNodeId(i));
577 
578  Uint32 currentRestartNodeIndex = 0;
579  UtilTransactions utilTrans(*ctx->getTab());
580  HugoTransactions hugoTrans(*ctx->getTab());
581 
582  {
583  int val = DumpStateOrd::DihMinTimeBetweenLCP;
584  if(restarter.dumpStateAllNodes(&val, 1) != 0){
585  g_err << "ERR: "<< step->getName()
586  << " failed on line " << __LINE__ << endl;
587  return NDBT_FAILED;
588  }
589  }
590 
591  while(i<=loops && result != NDBT_FAILED){
592 
593  g_info << "Loop " << i << "/"<< loops <<" started" << endl;
606  g_info << "Loading records..." << endl;
607  CHECK(hugoTrans.loadTable(pNdb, records) == 0);
608 
609  /*** 1 ***/
610  g_info << "1 - Stopping one node" << endl;
611  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
612  false,
613  true,
614  false) == 0);
615  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
616 
617  g_info << "Updating records..." << endl;
618  CHECK(hugoTrans.pkUpdateRecords(pNdb, records) == 0);
619 
620  g_info << "Restarting cluster..." << endl;
621  CHECK(restarter.restartAll() == 0);
622  CHECK(restarter.waitClusterStarted(timeout) == 0);
623  {
624  int val = DumpStateOrd::DihMinTimeBetweenLCP;
625  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
626  }
627  CHECK(pNdb->waitUntilReady(timeout) == 0);
628 
629  g_info << "Verifying records..." << endl;
630  CHECK(hugoTrans.pkReadRecords(pNdb, records) == 0);
631  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
632  CHECK(count == records);
633 
634  g_info << "2 - Stopping one node" << endl;
635  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
636  false,
637  true,
638  false) == 0);
639  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
640 
641  g_info << "Deleting 50% of records..." << endl;
642  CHECK(hugoTrans.pkDelRecords(pNdb, records/2) == 0);
643 
644  g_info << "Restarting cluster..." << endl;
645  CHECK(restarter.restartAll() == 0);
646  CHECK(restarter.waitClusterStarted(timeout) == 0);
647  {
648  int val = DumpStateOrd::DihMinTimeBetweenLCP;
649  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
650  }
651  CHECK(pNdb->waitUntilReady(timeout) == 0);
652 
653  g_info << "Verifying records..." << endl;
654  CHECK(hugoTrans.scanReadRecords(pNdb, records/2, 0, 64) == 0);
655  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
656  CHECK(count == (records/2));
657 
658  g_info << "3 - Stopping one node" << endl;
659  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
660  false,
661  true,
662  false) == 0);
663  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
664  g_info << "Deleting all records..." << endl;
665  CHECK(utilTrans.clearTable(pNdb, records/2) == 0);
666 
667  g_info << "Restarting cluster..." << endl;
668  CHECK(restarter.restartAll() == 0);
669  CHECK(restarter.waitClusterStarted(timeout) == 0);
670  {
671  int val = DumpStateOrd::DihMinTimeBetweenLCP;
672  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
673  }
674  CHECK(pNdb->waitUntilReady(timeout) == 0);
675 
676  ndbout << "Verifying records..." << endl;
677  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
678  CHECK(count == 0);
679 
680  i++;
681  }
682 
683  g_info << "runSystemRestart4 finished" << endl;
684 
685  return result;
686 }
687 
688 int runSystemRestart5(NDBT_Context* ctx, NDBT_Step* step){
689  Ndb* pNdb = GETNDB(step);
690  int result = NDBT_OK;
691  int timeout = 300;
692  Uint32 loops = ctx->getNumLoops();
693  int records = ctx->getNumRecords();
694  int count;
695  NdbRestarter restarter;
696  Uint32 i = 1;
697 
698  const Uint32 nodeCount = restarter.getNumDbNodes();
699  if(nodeCount < 2){
700  g_info << "SR5 - Needs atleast 2 nodes to test" << endl;
701  return NDBT_OK;
702  }
703 
704  Vector<int> nodeIds;
705  for(i = 0; i<nodeCount; i++)
706  nodeIds.push_back(restarter.getDbNodeId(i));
707 
708  Uint32 currentRestartNodeIndex = 0;
709  UtilTransactions utilTrans(*ctx->getTab());
710  HugoTransactions hugoTrans(*ctx->getTab());
711 
712  {
713  int val = DumpStateOrd::DihMinTimeBetweenLCP;
714  if(restarter.dumpStateAllNodes(&val, 1) != 0){
715  g_err << "ERR: "<< step->getName()
716  << " failed on line " << __LINE__ << endl;
717  return NDBT_FAILED;
718  }
719  }
720 
721  while(i<=loops && result != NDBT_FAILED){
722 
723  g_info << "Loop " << i << "/"<< loops <<" started" << endl;
736  g_info << "Loading records..." << endl;
737  hugoTrans.loadTable(pNdb, records);
738 
739  /*** 1 ***/
740  g_info << "1 - Stopping one node" << endl;
741  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
742  false,
743  true,
744  false) == 0);
745  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
746 
747  g_info << "Updating records..." << endl;
748  hugoTrans.pkUpdateRecords(pNdb, records);
749 
750  g_info << "Restarting cluster..." << endl;
751  CHECK(restarter.restartAll(false, false, true) == 0);
752  CHECK(restarter.waitClusterStarted(timeout) == 0);
753  {
754  int val = DumpStateOrd::DihMinTimeBetweenLCP;
755  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
756  }
757  CHECK(pNdb->waitUntilReady(timeout) == 0);
758 
759  g_info << "Verifying records..." << endl;
760  hugoTrans.pkReadRecords(pNdb, records);
761  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
762  //CHECK(count == records);
763 
764  g_info << "2 - Stopping one node" << endl;
765  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
766  false,
767  true,
768  false) == 0);
769  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
770 
771  g_info << "Deleting 50% of records..." << endl;
772  hugoTrans.pkDelRecords(pNdb, records/2);
773 
774  g_info << "Restarting cluster..." << endl;
775  CHECK(restarter.restartAll(false, false, true) == 0);
776  CHECK(restarter.waitClusterStarted(timeout) == 0);
777  {
778  int val = DumpStateOrd::DihMinTimeBetweenLCP;
779  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
780  }
781  CHECK(pNdb->waitUntilReady(timeout) == 0);
782 
783  g_info << "Verifying records..." << endl;
784  hugoTrans.scanReadRecords(pNdb, records/2, 0, 64);
785  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
786  //CHECK(count == (records/2));
787 
788  g_info << "3 - Stopping one node" << endl;
789  CHECK(restarter.restartOneDbNode(nodeIds[currentRestartNodeIndex],
790  false,
791  true,
792  false) == 0);
793  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
794  g_info << "Deleting all records..." << endl;
795  utilTrans.clearTable(pNdb, records/2);
796 
797  g_info << "Restarting cluster..." << endl;
798  CHECK(restarter.restartAll(false, false, true) == 0);
799  CHECK(restarter.waitClusterStarted(timeout) == 0);
800  {
801  int val = DumpStateOrd::DihMinTimeBetweenLCP;
802  CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
803  }
804  CHECK(pNdb->waitUntilReady(timeout) == 0);
805 
806  ndbout << "Verifying records..." << endl;
807  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
808  //CHECK(count == 0);
809 
810  CHECK(utilTrans.clearTable(pNdb) == 0);
811  i++;
812  }
813 
814  g_info << "runSystemRestart5 finished" << endl;
815 
816  return result;
817 }
818 
819 int runSystemRestart6(NDBT_Context* ctx, NDBT_Step* step){
820  Ndb* pNdb = GETNDB(step);
821  int result = NDBT_OK;
822  int timeout = 300;
823  Uint32 loops = ctx->getNumLoops();
824  int records = ctx->getNumRecords();
825  NdbRestarter restarter;
826  Uint32 i = 1;
827 
828  const Uint32 nodeCount = restarter.getNumDbNodes();
829  if(nodeCount < 2){
830  g_info << "SR6 - Needs atleast 2 nodes to test" << endl;
831  return NDBT_OK;
832  }
833 
834  Vector<int> nodeIds;
835  for(i = 0; i<nodeCount; i++)
836  nodeIds.push_back(restarter.getDbNodeId(i));
837 
838  Uint32 currentRestartNodeIndex = 0;
839  UtilTransactions utilTrans(*ctx->getTab());
840  HugoTransactions hugoTrans(*ctx->getTab());
841 
842  while(i<=loops && result != NDBT_FAILED){
843 
844  g_info << "Loop " << i << "/"<< loops <<" started" << endl;
851  g_info << "Loading records..." << endl;
852  hugoTrans.loadTable(pNdb, records);
853 
854  CHECK(restarter.restartAll(false, true, false) == 0);
855 
856  Uint32 nodeId = nodeIds[currentRestartNodeIndex];
857  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
858 
859  CHECK(restarter.restartOneDbNode(nodeId, true, true,false) == 0);
860  CHECK(restarter.waitClusterNoStart(timeout) == 0);
861  CHECK(restarter.startAll() == 0);
862  CHECK(restarter.waitClusterStarted(timeout) == 0);
863  CHECK(pNdb->waitUntilReady(timeout) == 0);
864  int count = records - 1;
865  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
866  CHECK(count == records);
867  CHECK(utilTrans.clearTable(pNdb) == 0);
868  i++;
869  }
870 
871  g_info << "runSystemRestart6 finished" << endl;
872 
873  return result;
874 }
875 
876 int runSystemRestart7(NDBT_Context* ctx, NDBT_Step* step){
877  Ndb* pNdb = GETNDB(step);
878  int result = NDBT_OK;
879  Uint32 loops = ctx->getNumLoops();
880  int records = ctx->getNumRecords();
881  NdbRestarter restarter;
882  Uint32 i = 1;
883 
884  const Uint32 nodeCount = restarter.getNumDbNodes();
885  if(nodeCount < 2){
886  g_info << "SR7 - Needs atleast 2 nodes to test" << endl;
887  return NDBT_OK;
888  }
889 
890  Vector<int> nodeIds;
891  for(i = 0; i<nodeCount; i++)
892  nodeIds.push_back(restarter.getDbNodeId(i));
893 
894  int a_nodeIds[64];
895  if(nodeCount > 64)
896  abort();
897 
898  Uint32 currentRestartNodeIndex = 1;
899  UtilTransactions utilTrans(*ctx->getTab());
900  HugoTransactions hugoTrans(*ctx->getTab());
901 
902  while(i<=loops && result != NDBT_FAILED){
903 
904  g_info << "Loop " << i << "/"<< loops <<" started" << endl;
913  g_info << "Loading records..." << endl;
914  hugoTrans.loadTable(pNdb, records);
915 
916  CHECK(restarter.restartAll(false, true, false) == 0);
917 
918  int nodeId = nodeIds[currentRestartNodeIndex];
919  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
920 
921  Uint32 j = 0;
922  for(Uint32 k = 0; k<nodeCount; k++){
923  if(nodeIds[k] != nodeId){
924  a_nodeIds[j++] = nodeIds[k];
925  }
926  }
927 
928  CHECK(restarter.startNodes(a_nodeIds, nodeCount - 1) == 0);
929  CHECK(restarter.waitNodesStarted(a_nodeIds, nodeCount - 1, 120) == 0);
930  CHECK(pNdb->waitUntilReady(5) == 0);
931  int count = records - 1;
932  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
933  CHECK(count == records);
934 
935  CHECK(restarter.startNodes(&nodeId, 1) == 0);
936  CHECK(restarter.waitNodesStarted(&nodeId, 1, 120) == 0);
937 
938  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
939  CHECK(count == records);
940  CHECK(utilTrans.clearTable(pNdb) == 0);
941 
942  i++;
943  }
944 
945  g_info << "runSystemRestart7 finished" << endl;
946 
947  return result;
948 }
949 
950 int runSystemRestart8(NDBT_Context* ctx, NDBT_Step* step){
951  Ndb* pNdb = GETNDB(step);
952  int result = NDBT_OK;
953  int timeout = 300;
954  Uint32 loops = ctx->getNumLoops();
955  int records = ctx->getNumRecords();
956  NdbRestarter restarter;
957  Uint32 i = 1;
958 
959  const Uint32 nodeCount = restarter.getNumDbNodes();
960  if(nodeCount < 2){
961  g_info << "SR8 - Needs atleast 2 nodes to test" << endl;
962  return NDBT_OK;
963  }
964 
965  Vector<int> nodeIds;
966  for(i = 0; i<nodeCount; i++)
967  nodeIds.push_back(restarter.getDbNodeId(i));
968 
969  int a_nodeIds[64];
970  if(nodeCount > 64)
971  abort();
972 
973  Uint32 currentRestartNodeIndex = 1;
974  UtilTransactions utilTrans(*ctx->getTab());
975  HugoTransactions hugoTrans(*ctx->getTab());
976 
977  while(i<=loops && result != NDBT_FAILED){
978 
979  g_info << "Loop " << i << "/"<< loops <<" started" << endl;
988  g_info << "Loading records..." << endl;
989  hugoTrans.loadTable(pNdb, records);
990 
991  CHECK(restarter.restartAll(false, true, false) == 0);
992 
993  int nodeId = nodeIds[currentRestartNodeIndex];
994  currentRestartNodeIndex = (currentRestartNodeIndex + 1 ) % nodeCount;
995 
996  Uint32 j = 0;
997  for(Uint32 k = 0; k<nodeCount; k++){
998  if(nodeIds[k] != nodeId){
999  a_nodeIds[j++] = nodeIds[k];
1000  }
1001  }
1002 
1003  CHECK(restarter.startNodes(a_nodeIds, nodeCount-1) == 0);
1004  CHECK(restarter.waitNodesStartPhase(a_nodeIds, nodeCount-1, 3, 120) == 0);
1005  CHECK(restarter.startNodes(&nodeId, 1) == 0);
1006  CHECK(restarter.waitClusterStarted(timeout) == 0);
1007  CHECK(pNdb->waitUntilReady() == 0);
1008 
1009  int count = records - 1;
1010  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
1011  CHECK(count == records);
1012  CHECK(utilTrans.clearTable(pNdb) == 0);
1013  i++;
1014  }
1015 
1016  g_info << "runSystemRestart8 finished" << endl;
1017 
1018  return result;
1019 }
1020 
1021 int runSystemRestart9(NDBT_Context* ctx, NDBT_Step* step){
1022  Ndb* pNdb = GETNDB(step);
1023  int result = NDBT_OK;
1024  int timeout = 300;
1025  NdbRestarter restarter;
1026  Uint32 i = 1;
1027 
1028  UtilTransactions utilTrans(*ctx->getTab());
1029  HugoTransactions hugoTrans(*ctx->getTab());
1030 
1031  int args[] = { DumpStateOrd::DihMaxTimeBetweenLCP };
1032  int dump[] = { DumpStateOrd::DihStartLcpImmediately };
1033 
1034  do {
1035  CHECK(restarter.dumpStateAllNodes(args, 1) == 0);
1036 
1037  HugoOperations ops(* ctx->getTab());
1038  CHECK(ops.startTransaction(pNdb) == 0);
1039  for(i = 0; i<10; i++){
1040  CHECK(ops.pkInsertRecord(pNdb, i, 1, 1) == 0);
1041  CHECK(ops.execute_NoCommit(pNdb) == 0);
1042  }
1043  for(i = 0; i<10; i++){
1044  CHECK(ops.pkUpdateRecord(pNdb, i, 1) == 0);
1045  CHECK(ops.execute_NoCommit(pNdb) == 0);
1046  }
1047  NdbSleep_SecSleep(10);
1048  CHECK(restarter.dumpStateAllNodes(dump, 1) == 0);
1049  NdbSleep_SecSleep(10);
1050  CHECK(ops.execute_Commit(pNdb) == 0);
1051 
1052  CHECK(restarter.restartAll() == 0);
1053  CHECK(restarter.waitClusterStarted(timeout) == 0);
1054  CHECK(pNdb->waitUntilReady(timeout) == 0);
1055  ops.closeTransaction(pNdb);
1056  } while(0);
1057 
1058  g_info << "runSystemRestart9 finished" << endl;
1059 
1060  return result;
1061 }
1062 
1063 int runBug18385(NDBT_Context* ctx, NDBT_Step* step){
1064  NdbRestarter restarter;
1065  const Uint32 nodeCount = restarter.getNumDbNodes();
1066  if(nodeCount < 2){
1067  g_info << "Bug18385 - Needs atleast 2 nodes to test" << endl;
1068  return NDBT_OK;
1069  }
1070 
1071  int node1 = restarter.getDbNodeId(rand() % nodeCount);
1072  int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
1073 
1074  if (node1 == -1 || node2 == -1)
1075  return NDBT_OK;
1076 
1077  int dump[] = { DumpStateOrd::DihSetTimeBetweenGcp, 300 };
1078 
1079  int result = NDBT_OK;
1080  do {
1081  CHECK(restarter.dumpStateAllNodes(dump, 2) == 0);
1082  CHECK(restarter.restartOneDbNode(node1, false, true, false) == 0);
1083  NdbSleep_SecSleep(3);
1084  CHECK(restarter.restartAll(false, true, false) == 0);
1085 
1086  Uint32 cnt = 0;
1087  int nodes[128];
1088  for(Uint32 i = 0; i<nodeCount; i++)
1089  if ((nodes[cnt] = restarter.getDbNodeId(i)) != node2)
1090  cnt++;
1091 
1092  assert(cnt == nodeCount - 1);
1093 
1094  CHECK(restarter.startNodes(nodes, cnt) == 0);
1095  CHECK(restarter.waitNodesStarted(nodes, cnt, 300) == 0);
1096 
1097  CHECK(restarter.insertErrorInNode(node2, 7170) == 0);
1098  CHECK(restarter.waitNodesNoStart(&node2, 1) == 0);
1099  CHECK(restarter.restartOneDbNode(node2, true, false, true) == 0);
1100  CHECK(restarter.waitNodesStarted(&node2, 1) == 0);
1101 
1102  } while(0);
1103 
1104  g_info << "Bug18385 finished" << endl;
1105 
1106  return result;
1107 }
1108 
1109 int runWaitStarted(NDBT_Context* ctx, NDBT_Step* step){
1110 
1111  NdbRestarter restarter;
1112  restarter.waitClusterStarted(300);
1113 
1114  NdbSleep_SecSleep(3);
1115  return NDBT_OK;
1116 }
1117 
1118 int runClearTable(NDBT_Context* ctx, NDBT_Step* step){
1119  int records = ctx->getNumRecords();
1120 
1121  Ndb* pNdb = GETNDB(step);
1122  if(pNdb->waitUntilReady(5) != 0){
1123  return NDBT_FAILED;
1124  }
1125 
1126  UtilTransactions utilTrans(*ctx->getTab());
1127  if (utilTrans.clearTable2(pNdb, records) != 0){
1128  return NDBT_FAILED;
1129  }
1130  return NDBT_OK;
1131 }
1132 
1133 int
1134 runBug21536(NDBT_Context* ctx, NDBT_Step* step)
1135 {
1136  NdbRestarter restarter;
1137  const Uint32 nodeCount = restarter.getNumDbNodes();
1138  if(nodeCount != 2){
1139  g_info << "Bug21536 - 2 nodes to test" << endl;
1140  return NDBT_OK;
1141  }
1142 
1143  int node1 = restarter.getDbNodeId(rand() % nodeCount);
1144  int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
1145 
1146  if (node1 == -1 || node2 == -1)
1147  return NDBT_OK;
1148 
1149  int result = NDBT_OK;
1150  do {
1151  CHECK(restarter.restartOneDbNode(node1, false, true, true) == 0);
1152  CHECK(restarter.waitNodesNoStart(&node1, 1) == 0);
1153  CHECK(restarter.insertErrorInNode(node1, 1000) == 0);
1154  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1155  CHECK(restarter.dumpStateOneNode(node1, val2, 2) == 0);
1156  CHECK(restarter.startNodes(&node1, 1) == 0);
1157  restarter.waitNodesStartPhase(&node1, 1, 3, 120);
1158  CHECK(restarter.waitNodesNoStart(&node1, 1) == 0);
1159 
1160  CHECK(restarter.restartOneDbNode(node2, true, true, true) == 0);
1161  CHECK(restarter.waitNodesNoStart(&node2, 1) == 0);
1162  CHECK(restarter.startNodes(&node1, 1) == 0);
1163  CHECK(restarter.waitNodesStarted(&node1, 1) == 0);
1164  CHECK(restarter.startNodes(&node2, 1) == 0);
1165  CHECK(restarter.waitClusterStarted() == 0);
1166 
1167  } while(0);
1168 
1169  g_info << "Bug21536 finished" << endl;
1170 
1171  return result;
1172 }
1173 
1174 int
1175 runBug24664(NDBT_Context* ctx, NDBT_Step* step)
1176 {
1177  int result = NDBT_OK;
1178  NdbRestarter restarter;
1179  Ndb* pNdb = GETNDB(step);
1180 
1181  int records = ctx->getNumRecords();
1182  UtilTransactions utilTrans(*ctx->getTab());
1183  HugoTransactions hugoTrans(*ctx->getTab());
1184 
1185  int args[] = { DumpStateOrd::DihMaxTimeBetweenLCP };
1186  int dump[] = { DumpStateOrd::DihStartLcpImmediately };
1187 
1188  restarter.getNumDbNodes();
1189  int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
1190  NdbLogEventHandle handle =
1191  ndb_mgm_create_logevent_handle(restarter.handle, filter);
1192 
1193  struct ndb_logevent event;
1194 
1195  do {
1196  CHECK(restarter.dumpStateAllNodes(args, 1) == 0);
1197  CHECK(restarter.dumpStateAllNodes(dump, 1) == 0);
1198  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1199  event.type != NDB_LE_LocalCheckpointStarted);
1200  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1201  event.type != NDB_LE_LocalCheckpointCompleted);
1202 
1203  if (hugoTrans.loadTable(GETNDB(step), records) != 0){
1204  return NDBT_FAILED;
1205  }
1206 
1207  restarter.insertErrorInAllNodes(10039); // Hang LCP
1208  CHECK(restarter.dumpStateAllNodes(dump, 1) == 0);
1209  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1210  event.type != NDB_LE_LocalCheckpointStarted);
1211  NdbSleep_SecSleep(3);
1212  CHECK(utilTrans.clearTable(pNdb, records) == 0);
1213  if (hugoTrans.loadTable(GETNDB(step), records) != 0){
1214  return NDBT_FAILED;
1215  }
1216 
1217  restarter.insertErrorInAllNodes(10040); // Resume LCP
1218  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1219  event.type != NDB_LE_LocalCheckpointCompleted);
1220 
1221  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1222  event.type != NDB_LE_GlobalCheckpointCompleted);
1223  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1224  event.type != NDB_LE_GlobalCheckpointCompleted);
1225  restarter.restartAll(false, false, true);
1226  CHECK(restarter.waitClusterStarted() == 0);
1227  } while(false);
1228 
1229  return result;
1230 }
1231 
1232 int
1233 runBug27434(NDBT_Context* ctx, NDBT_Step* step)
1234 {
1235  int result = NDBT_OK;
1236  NdbRestarter restarter;
1237  const Uint32 nodeCount = restarter.getNumDbNodes();
1238 
1239  if (nodeCount < 2)
1240  return NDBT_OK;
1241 
1242  int args[] = { DumpStateOrd::DihMaxTimeBetweenLCP };
1243  int dump[] = { DumpStateOrd::DihStartLcpImmediately };
1244 
1245  int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
1246  NdbLogEventHandle handle =
1247  ndb_mgm_create_logevent_handle(restarter.handle, filter);
1248 
1249  struct ndb_logevent event;
1250 
1251  do {
1252  int node1 = restarter.getDbNodeId(rand() % nodeCount);
1253  CHECK(restarter.restartOneDbNode(node1, false, true, true) == 0);
1254  NdbSleep_SecSleep(3);
1255  CHECK(restarter.waitNodesNoStart(&node1, 1) == 0);
1256 
1257  CHECK(restarter.dumpStateAllNodes(args, 1) == 0);
1258 
1259  for (Uint32 i = 0; i<3; i++)
1260  {
1261  CHECK(restarter.dumpStateAllNodes(dump, 1) == 0);
1262  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1263  event.type != NDB_LE_LocalCheckpointStarted);
1264  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1265  event.type != NDB_LE_LocalCheckpointCompleted);
1266  }
1267 
1268  restarter.restartAll(false, true, true);
1269  NdbSleep_SecSleep(3);
1270  CHECK(restarter.waitClusterNoStart() == 0);
1271  restarter.insertErrorInNode(node1, 5046);
1272  restarter.startAll();
1273  CHECK(restarter.waitClusterStarted() == 0);
1274  } while(false);
1275 
1276  return result;
1277 }
1278 
1279 int
1280 runBug29167(NDBT_Context* ctx, NDBT_Step* step)
1281 {
1282  int result = NDBT_OK;
1283  NdbRestarter restarter;
1284  const Uint32 nodeCount = restarter.getNumDbNodes();
1285 
1286  if (nodeCount < 4)
1287  return NDBT_OK;
1288 
1289  struct ndb_logevent event;
1290  int master = restarter.getMasterNodeId();
1291  do {
1292  int node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
1293  int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
1294 
1295  ndbout_c("node1: %u node2: %u", node1, node2);
1296 
1297  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1298  restarter.dumpStateAllNodes(val2, 2);
1299  int dump[] = { DumpStateOrd::DihSetTimeBetweenGcp, 30000 };
1300  restarter.dumpStateAllNodes(dump, 2);
1301 
1302  int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
1303  NdbLogEventHandle handle =
1304  ndb_mgm_create_logevent_handle(restarter.handle, filter);
1305 
1306  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1307  event.type != NDB_LE_GlobalCheckpointCompleted);
1308 
1309  ndb_mgm_destroy_logevent_handle(&handle);
1310 
1311  CHECK(restarter.insertErrorInAllNodes(932) == 0);
1312 
1313  CHECK(restarter.insertErrorInNode(node1, 7183) == 0);
1314  CHECK(restarter.insertErrorInNode(node2, 7183) == 0);
1315 
1316  CHECK(restarter.waitClusterNoStart() == 0);
1317  restarter.startAll();
1318  CHECK(restarter.waitClusterStarted() == 0);
1319  } while(false);
1320 
1321  return result;
1322 }
1323 int
1324 runBug28770(NDBT_Context* ctx, NDBT_Step* step) {
1325  Ndb* pNdb = GETNDB(step);
1326  NdbRestarter restarter;
1327  int result = NDBT_OK;
1328  int count = 0;
1329  Uint32 i = 0;
1330  Uint32 loops = ctx->getNumLoops();
1331  int records = ctx->getNumRecords();
1332  UtilTransactions utilTrans(*ctx->getTab());
1333  HugoTransactions hugoTrans(*ctx->getTab());
1334 
1335  g_info << "Loading records..." << endl; hugoTrans.loadTable(pNdb,
1336  records);
1337 
1338 
1339  while(i<=loops && result != NDBT_FAILED)
1340  {
1341  g_info << "Loop " << i << "/"<< loops <<" started" << endl;
1342  if (i == 0)
1343  {
1344  CHECK(restarter.restartAll(false, true, false) == 0); // graceful
1345  }
1346  else
1347  {
1348  CHECK(restarter.restartAll(false, true, true) == 0); // abort
1349  }
1350  CHECK(restarter.waitClusterNoStart() == 0);
1351  restarter.insertErrorInAllNodes(6024);
1352  CHECK(restarter.startAll()== 0);
1353  CHECK(restarter.waitClusterStarted() == 0);
1354  CHECK(pNdb->waitUntilReady() == 0);
1355  CHECK(utilTrans.selectCount(pNdb, 64, &count) == 0);
1356  CHECK(count == records);
1357  i++;
1358  }
1359  ndbout << " runBug28770 finished" << endl;
1360  return result;
1361 }
1362 
1363 int
1364 runStopper(NDBT_Context* ctx, NDBT_Step* step)
1365 {
1366  NdbRestarter restarter;
1367  Uint32 stop = 0;
1368 loop:
1369  while (!ctx->isTestStopped() &&
1370  ((stop = ctx->getProperty("StopAbort", Uint32(0))) == 0))
1371  {
1372  NdbSleep_MilliSleep(30);
1373  }
1374 
1375  if (ctx->isTestStopped())
1376  {
1377  return NDBT_OK;
1378  }
1379 
1380  ctx->setProperty("StopAbort", Uint32(0));
1381 
1382  ndbout << "Killing in " << stop << "ms..." << flush;
1383  NdbSleep_MilliSleep(stop);
1384  restarter.restartAll(false, true, true);
1385  goto loop;
1386 }
1387 
1388 int runSR_DD_1(NDBT_Context* ctx, NDBT_Step* step)
1389 {
1390  Ndb* pNdb = GETNDB(step);
1391  int result = NDBT_OK;
1392  Uint32 loops = ctx->getNumLoops();
1393  NdbRestarter restarter;
1394  NdbBackup backup(GETNDB(step)->getNodeId()+1);
1395  bool lcploop = ctx->getProperty("LCP", (unsigned)0);
1396  bool all = ctx->getProperty("ALL", (unsigned)0);
1397 
1398  Uint32 i = 1;
1399 
1400  int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1401  int lcp = DumpStateOrd::DihMinTimeBetweenLCP;
1402 
1403  int startFrom = 0;
1404 
1405  HugoTransactions hugoTrans(*ctx->getTab());
1406  while(i<=loops && result != NDBT_FAILED)
1407  {
1408 
1409  if (lcploop)
1410  {
1411  CHECK(restarter.dumpStateAllNodes(&lcp, 1) == 0);
1412  }
1413 
1414  int nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
1415  //CHECK(restarter.dumpStateAllNodes(&val, 1) == 0);
1416 
1417  ndbout << "Loop " << i << "/"<< loops <<" started" << endl;
1418  ndbout << "Loading records..." << startFrom << endl;
1419  CHECK(hugoTrans.loadTable(pNdb, startFrom) == 0);
1420 
1421  if (!all)
1422  {
1423  ndbout << "Making " << nodeId << " crash" << endl;
1424  int kill[] = { 9999, 1000, 3000 };
1425  CHECK(restarter.dumpStateOneNode(nodeId, val, 2) == 0);
1426  CHECK(restarter.dumpStateOneNode(nodeId, kill, 3) == 0);
1427  }
1428  else
1429  {
1430  ndbout << "Crashing cluster" << endl;
1431  ctx->setProperty("StopAbort", 1000 + rand() % (3000 - 1000));
1432  }
1433  Uint64 end = NdbTick_CurrentMillisecond() + 4000;
1434  Uint32 row = startFrom;
1435  do {
1436  ndbout << "Loading from " << row << " to " << row + 1000 << endl;
1437  if (hugoTrans.loadTableStartFrom(pNdb, row, 1000) != 0)
1438  break;
1439  row += 1000;
1440  } while (NdbTick_CurrentMillisecond() < end);
1441 
1442  if (!all)
1443  {
1444  ndbout << "Waiting for " << nodeId << " to restart" << endl;
1445  CHECK(restarter.waitNodesNoStart(&nodeId, 1) == 0);
1446  ndbout << "Restarting cluster" << endl;
1447  CHECK(restarter.restartAll(false, true, true) == 0);
1448  }
1449  else
1450  {
1451  ndbout << "Waiting for cluster to restart" << endl;
1452  }
1453  CHECK(restarter.waitClusterNoStart() == 0);
1454  CHECK(restarter.startAll() == 0);
1455  CHECK(restarter.waitClusterStarted() == 0);
1456  CHECK(pNdb->waitUntilReady() == 0);
1457 
1458  ndbout << "Starting backup..." << flush;
1459  CHECK(backup.start() == 0);
1460  ndbout << "done" << endl;
1461 
1462  int cnt = 0;
1463  CHECK(hugoTrans.selectCount(pNdb, 0, &cnt) == 0);
1464  ndbout << "Found " << cnt << " records..." << endl;
1465  ndbout << "Updating..." << endl;
1466  CHECK(hugoTrans.scanUpdateRecords(pNdb,
1467  NdbScanOperation::SF_TupScan, cnt) == 0);
1468  ndbout << "Clearing..." << endl;
1469  CHECK(hugoTrans.clearTable(pNdb,
1470  NdbScanOperation::SF_TupScan, cnt) == 0);
1471 
1472  if (cnt > startFrom)
1473  {
1474  startFrom = cnt;
1475  }
1476  startFrom += 1000;
1477  i++;
1478  }
1479 
1480  ndbout << "runSR_DD_1 finished" << endl;
1481  ctx->stopTest();
1482  return result;
1483 }
1484 
1485 int runSR_DD_2(NDBT_Context* ctx, NDBT_Step* step)
1486 {
1487  Ndb* pNdb = GETNDB(step);
1488  int result = NDBT_OK;
1489  Uint32 loops = ctx->getNumLoops();
1490  Uint32 rows = ctx->getNumRecords();
1491  NdbRestarter restarter;
1492  NdbBackup backup(GETNDB(step)->getNodeId()+1);
1493  bool lcploop = ctx->getProperty("LCP", (unsigned)0);
1494  bool all = ctx->getProperty("ALL", (unsigned)0);
1495  int error = (int)ctx->getProperty("ERROR", (unsigned)0);
1496  rows = ctx->getProperty("ROWS", rows);
1497 
1498  Uint32 i = 1;
1499 
1500  int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1501  int lcp = DumpStateOrd::DihMinTimeBetweenLCP;
1502 
1503  if (error)
1504  {
1505  restarter.insertErrorInAllNodes(error);
1506  }
1507 
1508  HugoTransactions hugoTrans(*ctx->getTab());
1509  while(i<=loops && result != NDBT_FAILED)
1510  {
1511 
1512  if (lcploop)
1513  {
1514  CHECK(restarter.dumpStateAllNodes(&lcp, 1) == 0);
1515  }
1516 
1517  int nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
1518 
1519  if (!all)
1520  {
1521  ndbout << "Making " << nodeId << " crash" << endl;
1522  int kill[] = { 9999, 3000, 10000 };
1523  CHECK(restarter.dumpStateOneNode(nodeId, val, 2) == 0);
1524  CHECK(restarter.dumpStateOneNode(nodeId, kill, 3) == 0);
1525  }
1526  else
1527  {
1528  ndbout << "Crashing cluster" << endl;
1529  ctx->setProperty("StopAbort", 3000 + rand() % (10000 - 3000));
1530  }
1531 
1532  Uint64 end = NdbTick_CurrentMillisecond() + 11000;
1533  do {
1534  if (hugoTrans.loadTable(pNdb, rows) != 0)
1535  break;
1536 
1537  if (hugoTrans.clearTable(pNdb, NdbScanOperation::SF_TupScan, rows) != 0)
1538  break;
1539  } while (NdbTick_CurrentMillisecond() < end);
1540 
1541  if (!all)
1542  {
1543  ndbout << "Waiting for " << nodeId << " to restart" << endl;
1544  CHECK(restarter.waitNodesNoStart(&nodeId, 1) == 0);
1545  ndbout << "Restarting cluster" << endl;
1546  CHECK(restarter.restartAll(false, true, true) == 0);
1547  }
1548  else
1549  {
1550  ndbout << "Waiting for cluster to restart" << endl;
1551  }
1552 
1553  CHECK(restarter.waitClusterNoStart() == 0);
1554  CHECK(restarter.startAll() == 0);
1555  CHECK(restarter.waitClusterStarted() == 0);
1556  CHECK(pNdb->waitUntilReady() == 0);
1557 
1558  if (error)
1559  {
1560  restarter.insertErrorInAllNodes(error);
1561  }
1562 
1563  ndbout << "Starting backup..." << flush;
1564  CHECK(backup.start() == 0);
1565  ndbout << "done" << endl;
1566 
1567  int cnt = 0;
1568  CHECK(hugoTrans.selectCount(pNdb, 0, &cnt) == 0);
1569  ndbout << "Found " << cnt << " records..." << endl;
1570  ndbout << "Updating..." << endl;
1571  CHECK(hugoTrans.scanUpdateRecords(pNdb,
1572  NdbScanOperation::SF_TupScan, cnt) == 0);
1573  ndbout << "Clearing..." << endl;
1574  CHECK(hugoTrans.clearTable(pNdb,
1575  NdbScanOperation::SF_TupScan, cnt) == 0);
1576  i++;
1577  }
1578 
1579  if (error)
1580  {
1581  restarter.insertErrorInAllNodes(0);
1582  }
1583 
1584  ndbout << "runSR_DD_2 finished" << endl;
1585  ctx->stopTest();
1586  return result;
1587 }
1588 
1589 int runSR_DD_3(NDBT_Context* ctx, NDBT_Step* step)
1590 {
1591  Ndb* pNdb = GETNDB(step);
1592  int result = NDBT_OK;
1593  Uint32 loops = ctx->getNumLoops();
1594  Uint32 rows = ctx->getNumRecords();
1595  NdbRestarter restarter;
1596  NdbBackup backup(GETNDB(step)->getNodeId()+1);
1597  bool lcploop = ctx->getProperty("LCP", (unsigned)0);
1598  bool all = ctx->getProperty("ALL", (unsigned)0);
1599  int error = (int)ctx->getProperty("ERROR", (unsigned)0);
1600  rows = ctx->getProperty("ROWS", rows);
1601 
1602  Uint32 i = 1;
1603 
1604  int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1605  int lcp = DumpStateOrd::DihMinTimeBetweenLCP;
1606 
1607  if (error)
1608  {
1609  restarter.insertErrorInAllNodes(error);
1610  }
1611 
1612  HugoTransactions hugoTrans(*ctx->getTab());
1613  while(i<=loops && result != NDBT_FAILED)
1614  {
1615 
1616  if (lcploop)
1617  {
1618  CHECK(restarter.dumpStateAllNodes(&lcp, 1) == 0);
1619  }
1620 
1621  int nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
1622 
1623  if (hugoTrans.loadTable(pNdb, rows) != 0)
1624  {
1625  return NDBT_FAILED;
1626  }
1627 
1628  if (!all)
1629  {
1630  ndbout << "Making " << nodeId << " crash" << endl;
1631  int kill[] = { 9999, 3000, 10000 };
1632  CHECK(restarter.dumpStateOneNode(nodeId, val, 2) == 0);
1633  CHECK(restarter.dumpStateOneNode(nodeId, kill, 3) == 0);
1634  }
1635  else
1636  {
1637  ndbout << "Crashing cluster" << endl;
1638  ctx->setProperty("StopAbort", 3000 + rand() % (10000 - 3000));
1639  }
1640 
1641  int deletedrows[100];
1642  Uint64 end = NdbTick_CurrentMillisecond() + 13000;
1643  do {
1644  Uint32 cnt = 0;
1645  for (; cnt<NDB_ARRAY_SIZE(deletedrows); cnt++)
1646  {
1647  deletedrows[cnt] = rand() % rows;
1648  if (hugoTrans.startTransaction(pNdb))
1649  break;
1650  if (hugoTrans.pkDeleteRecord(pNdb, deletedrows[cnt]))
1651  break;
1652  if (hugoTrans.execute_Commit(pNdb))
1653  break;
1654  hugoTrans.closeTransaction(pNdb);
1655  }
1656  if (hugoTrans.getTransaction() != 0)
1657  hugoTrans.closeTransaction(pNdb);
1658 
1659  if (hugoTrans.scanUpdateRecords(pNdb, NdbScanOperation::SF_TupScan,0)!=0)
1660  break;
1661 
1662  for (Uint32 n = 0; n<cnt; n++)
1663  {
1664  if (hugoTrans.startTransaction(pNdb))
1665  break;
1666  if (hugoTrans.pkInsertRecord(pNdb, deletedrows[n], 1, rand()))
1667  break;
1668  if (hugoTrans.execute_Commit(pNdb))
1669  break;
1670  hugoTrans.closeTransaction(pNdb);
1671  }
1672  if (hugoTrans.getTransaction() != 0)
1673  hugoTrans.closeTransaction(pNdb);
1674 
1675  if (hugoTrans.scanUpdateRecords(pNdb, NdbScanOperation::SF_TupScan,0)!=0)
1676  break;
1677  } while (NdbTick_CurrentMillisecond() < end);
1678 
1679  if (!all)
1680  {
1681  ndbout << "Waiting for " << nodeId << " to restart" << endl;
1682  CHECK(restarter.waitNodesNoStart(&nodeId, 1) == 0);
1683  ndbout << "Restarting cluster" << endl;
1684  CHECK(restarter.restartAll(false, true, true) == 0);
1685  }
1686  else
1687  {
1688  ndbout << "Waiting for cluster to restart" << endl;
1689  }
1690 
1691  CHECK(restarter.waitClusterNoStart() == 0);
1692  CHECK(restarter.startAll() == 0);
1693  CHECK(restarter.waitClusterStarted() == 0);
1694  if (error)
1695  {
1696  restarter.insertErrorInAllNodes(error);
1697  }
1698 
1699  ndbout << "Starting backup..." << flush;
1700  CHECK(backup.start() == 0);
1701  ndbout << "done" << endl;
1702 
1703  int cnt = 0;
1704  CHECK(hugoTrans.selectCount(pNdb, 0, &cnt) == 0);
1705  ndbout << "Found " << cnt << " records..." << endl;
1706  ndbout << "Updating..." << endl;
1707  CHECK(hugoTrans.scanUpdateRecords(pNdb,
1708  NdbScanOperation::SF_TupScan, cnt) == 0);
1709  ndbout << "Clearing..." << endl;
1710  CHECK(hugoTrans.clearTable(pNdb,
1711  NdbScanOperation::SF_TupScan, cnt) == 0);
1712  i++;
1713  }
1714 
1715  if (error)
1716  {
1717  restarter.insertErrorInAllNodes(0);
1718  }
1719 
1720  ndbout << "runSR_DD_3 finished" << endl;
1721  ctx->stopTest();
1722  return result;
1723 }
1724 
1725 int runBug22696(NDBT_Context* ctx, NDBT_Step* step)
1726 {
1727  Ndb* pNdb = GETNDB(step);
1728  int result = NDBT_OK;
1729  Uint32 loops = ctx->getNumLoops();
1730  Uint32 rows = ctx->getNumRecords();
1731  NdbRestarter restarter;
1732  HugoTransactions hugoTrans(*ctx->getTab());
1733 
1734  Uint32 i = 0;
1735  while(i<=loops && result != NDBT_FAILED)
1736  {
1737  ndbout_c("loop %u", i);
1738  for (Uint32 j = 0; j<10 && result != NDBT_FAILED; j++)
1739  CHECK(hugoTrans.scanUpdateRecords(pNdb, rows) == 0);
1740 
1741  CHECK(restarter.restartAll(false, true, i > 0 ? true : false) == 0);
1742  CHECK(restarter.waitClusterNoStart() == 0);
1743  CHECK(restarter.insertErrorInAllNodes(7072) == 0);
1744  CHECK(restarter.startAll() == 0);
1745  CHECK(restarter.waitClusterStarted() == 0);
1746  CHECK(pNdb->waitUntilReady() == 0);
1747 
1748  i++;
1749  if (i < loops)
1750  {
1751  NdbSleep_SecSleep(5); // Wait for a few gcp
1752  }
1753  }
1754 
1755  ctx->stopTest();
1756  return result;
1757 }
1758 
1759 int
1760 runCreateAllTables(NDBT_Context* ctx, NDBT_Step* step)
1761 {
1762  if (NDBT_Tables::createAllTables(GETNDB(step), false, true))
1763  return NDBT_FAILED;
1764  return NDBT_OK;
1765 }
1766 
1767 int
1768 runBasic(NDBT_Context* ctx, NDBT_Step* step)
1769 {
1770  Ndb* pNdb = GETNDB(step);
1771  NdbDictionary::Dictionary * pDict = pNdb->getDictionary();
1772  int loops = ctx->getNumLoops();
1773  int records = ctx->getNumRecords();
1774  NdbRestarter restarter;
1775  int result = NDBT_OK;
1776 
1777  for (int l = 0; l<loops; l++)
1778  {
1779  for (int i = 0; i<NDBT_Tables::getNumTables(); i++)
1780  {
1781  const NdbDictionary::Table* tab =
1782  pDict->getTable(NDBT_Tables::getTable(i)->getName());
1783  HugoTransactions trans(* tab);
1784  switch(l % 3){
1785  case 0:
1786  trans.loadTable(pNdb, records);
1787  trans.scanUpdateRecords(pNdb, records);
1788  break;
1789  case 1:
1790  trans.scanUpdateRecords(pNdb, records);
1791  trans.clearTable(pNdb, records/2);
1792  trans.loadTable(pNdb, records/2);
1793  break;
1794  case 2:
1795  trans.clearTable(pNdb, records/2);
1796  trans.loadTable(pNdb, records/2);
1797  trans.clearTable(pNdb, records/2);
1798  break;
1799  }
1800  }
1801 
1802  ndbout << "Restarting cluster..." << endl;
1803  CHECK(restarter.restartAll(false, true, false) == 0);
1804  CHECK(restarter.waitClusterNoStart() == 0);
1805  CHECK(restarter.startAll() == 0);
1806  CHECK(restarter.waitClusterStarted() == 0);
1807  CHECK(pNdb->waitUntilReady() == 0);
1808 
1809  for (int i = 0; i<NDBT_Tables::getNumTables(); i++)
1810  {
1811  const NdbDictionary::Table* tab =
1812  pDict->getTable(NDBT_Tables::getTable(i)->getName());
1813  HugoTransactions trans(* tab);
1814  trans.scanUpdateRecords(pNdb, records);
1815  }
1816  }
1817 
1818  return result;
1819 }
1820 
1821 int
1822 runDropAllTables(NDBT_Context* ctx, NDBT_Step* step)
1823 {
1824  NDBT_Tables::dropAllTables(GETNDB(step));
1825  return NDBT_OK;
1826 }
1827 
1828 int
1829 runTO(NDBT_Context* ctx, NDBT_Step* step)
1830 {
1831  Ndb* pNdb = GETNDB(step);
1832  int result = NDBT_OK;
1833  Uint32 loops = ctx->getNumLoops();
1834  Uint32 rows = ctx->getNumRecords();
1835  NdbRestarter res;
1836  HugoTransactions hugoTrans(*ctx->getTab());
1837 
1838  if (res.getNumDbNodes() < 2)
1839  return NDBT_OK;
1840 
1841  Uint32 nodeGroups[256];
1842  Bitmask<256/32> nodeGroupMap;
1843  for (int j = 0; j<res.getNumDbNodes(); j++)
1844  {
1845  int node = res.getDbNodeId(j);
1846  nodeGroups[node] = res.getNodeGroup(node);
1847  nodeGroupMap.set(nodeGroups[node]);
1848  }
1849 
1850  struct ndb_logevent event;
1851  int val[] = { DumpStateOrd::DihMinTimeBetweenLCP, 0 };
1852 
1853  Uint32 i = 0;
1854  while(i<=loops && result != NDBT_FAILED)
1855  {
1856  CHECK(res.dumpStateAllNodes(val, 1) == 0);
1857 
1858  int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
1859  NdbLogEventHandle handle =
1860  ndb_mgm_create_logevent_handle(res.handle, filter);
1861 
1862  Bitmask<256/32> notstopped = nodeGroupMap;
1863  while(!notstopped.isclear())
1864  {
1865  int node;
1866  do {
1867  node = res.getDbNodeId(rand() % res.getNumDbNodes());
1868  } while (!notstopped.get(nodeGroups[node]));
1869 
1870  notstopped.clear(nodeGroups[node]);
1871  ndbout_c("stopping %u", node);
1872  CHECK(res.restartOneDbNode(node, false, true, true) == 0);
1873  CHECK(res.waitNodesNoStart(&node, 1) == 0);
1874  for (Uint32 j = 0; j<25; j++)
1875  {
1876  if (! (hugoTrans.scanUpdateRecords(pNdb, 0) == 0))
1877  break;
1878  }
1879  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1880  event.type != NDB_LE_LocalCheckpointCompleted);
1881  }
1882 
1883  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1884  event.type != NDB_LE_LocalCheckpointCompleted);
1885 
1886  Uint32 LCP = event.LocalCheckpointCompleted.lci;
1887  ndbout_c("LCP: %u", LCP);
1888 
1889  do
1890  {
1891  bzero(&event, sizeof(event));
1892  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1893  event.type != NDB_LE_LocalCheckpointCompleted)
1894  bzero(&event, sizeof(event));
1895 
1896  if (event.type == NDB_LE_LocalCheckpointCompleted &&
1897  event.LocalCheckpointCompleted.lci < LCP + 3)
1898  {
1899  hugoTrans.scanUpdateRecords(pNdb, 0);
1900  }
1901  else
1902  {
1903  break;
1904  }
1905  } while (true);
1906 
1907  ndbout_c("LCP: %u", event.LocalCheckpointCompleted.lci);
1908 
1909  CHECK(res.restartAll(false, true, true) == 0);
1910  CHECK(res.waitClusterNoStart() == 0);
1911  CHECK(res.startAll() == 0);
1912  NDB_TICKS now = NdbTick_CurrentMillisecond();
1918  NullOutputStream null;
1919  OutputStream * save[1];
1920  save[0] = g_err.m_out;
1921  g_err.m_out = &null;
1922  do
1923  {
1924  hugoTrans.scanUpdateRecords(pNdb, 0);
1925  } while (NdbTick_CurrentMillisecond() < (now + 30000));
1926  g_err.m_out = save[0];
1927  CHECK(res.waitClusterStarted() == 0);
1928  CHECK(pNdb->waitUntilReady() == 0);
1929 
1930  hugoTrans.clearTable(pNdb);
1931  hugoTrans.loadTable(pNdb, rows);
1932 
1933  CHECK(res.dumpStateAllNodes(val, 1) == 0);
1934 
1935  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1936  event.type != NDB_LE_LocalCheckpointCompleted);
1937 
1938  ndb_mgm_destroy_logevent_handle(&handle);
1939 
1940  i++;
1941  }
1942 
1943  res.dumpStateAllNodes(val, 2); // Reset LCP time
1944 
1945  ctx->stopTest();
1946  return result;
1947 }
1948 
1949 int runBug45154(NDBT_Context* ctx, NDBT_Step* step)
1950 {
1951  Ndb* pNdb = GETNDB(step);
1952  NdbDictionary::Dictionary * pDict = pNdb->getDictionary();
1953  int result = NDBT_OK;
1954  Uint32 loops = ctx->getNumLoops();
1955  Uint32 rows = ctx->getNumRecords();
1956  NdbRestarter restarter;
1957 
1958  restarter.getNumDbNodes();
1959  int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
1960  NdbLogEventHandle handle =
1961  ndb_mgm_create_logevent_handle(restarter.handle, filter);
1962 
1963  struct ndb_logevent event;
1964 
1965  Uint32 frag_data[128];
1966  bzero(frag_data, sizeof(frag_data));
1967 
1968  NdbDictionary::HashMap map;
1969  pDict->getDefaultHashMap(map, 2*restarter.getNumDbNodes());
1970  pDict->createHashMap(map);
1971 
1972  pDict->getDefaultHashMap(map, restarter.getNumDbNodes());
1973  pDict->createHashMap(map);
1974 
1975  for(Uint32 i = 0; i < loops && result != NDBT_FAILED; i++)
1976  {
1977  ndbout_c("loop %u", i);
1978 
1979  NdbDictionary::Table copy = *ctx->getTab();
1980  copy.setName("BUG_45154");
1981  copy.setFragmentType(NdbDictionary::Object::DistrKeyLin);
1982  copy.setFragmentCount(2 * restarter.getNumDbNodes());
1983  copy.setFragmentData(frag_data, 2*restarter.getNumDbNodes());
1984  pDict->dropTable("BUG_45154");
1985  int res = pDict->createTable(copy);
1986  if (res != 0)
1987  {
1988  ndbout << pDict->getNdbError() << endl;
1989  return NDBT_FAILED;
1990  }
1991  const NdbDictionary::Table* copyptr= pDict->getTable("BUG_45154");
1992 
1993  {
1994  HugoTransactions hugoTrans(*copyptr);
1995  hugoTrans.loadTable(pNdb, rows);
1996  }
1997 
1998  int dump[] = { DumpStateOrd::DihStartLcpImmediately };
1999  for (int l = 0; l<2; l++)
2000  {
2001  CHECK(restarter.dumpStateAllNodes(dump, 1) == 0);
2002  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
2003  event.type != NDB_LE_LocalCheckpointStarted);
2004  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
2005  event.type != NDB_LE_LocalCheckpointCompleted);
2006  }
2007 
2008  pDict->dropTable("BUG_45154");
2009  copy.setFragmentCount(restarter.getNumDbNodes());
2010  copy.setFragmentData(frag_data, restarter.getNumDbNodes());
2011  res = pDict->createTable(copy);
2012  if (res != 0)
2013  {
2014  ndbout << pDict->getNdbError() << endl;
2015  return NDBT_FAILED;
2016  }
2017  copyptr = pDict->getTable("BUG_45154");
2018 
2019  {
2020  HugoTransactions hugoTrans(*copyptr);
2021  hugoTrans.loadTable(pNdb, rows);
2022  for (Uint32 pp = 0; pp<3; pp++)
2023  hugoTrans.scanUpdateRecords(pNdb, rows);
2024  }
2025  restarter.restartAll(false, true, true);
2026  restarter.waitClusterNoStart();
2027  restarter.startAll();
2028  restarter.waitClusterStarted();
2029 
2030  pDict->dropTable("BUG_45154");
2031  }
2032 
2033  ctx->stopTest();
2034  return result;
2035 }
2036 
2037 int runBug46651(NDBT_Context* ctx, NDBT_Step* step)
2038 {
2039  Ndb* pNdb = GETNDB(step);
2040  NdbDictionary::Dictionary * pDict = pNdb->getDictionary();
2041  Uint32 rows = ctx->getNumRecords();
2042  NdbRestarter res;
2043 
2044  NdbDictionary::Table tab;
2045  tab.setName("BUG_46651");
2046 
2047  NdbDictionary::Column col;
2048  col.setName("ATTR1");
2049  col.setType(NdbDictionary::Column::Unsigned);
2050  col.setLength(1);
2051  col.setPrimaryKey(true);
2052  col.setNullable(false);
2053  col.setAutoIncrement(false);
2054  tab.addColumn(col);
2055  col.setName("ATTR2");
2056  col.setType(NdbDictionary::Column::Unsigned);
2057  col.setLength(1);
2058  col.setPrimaryKey(false);
2059  col.setNullable(false);
2060  tab.addColumn(col);
2061  col.setName("ATTR3");
2062  col.setType(NdbDictionary::Column::Unsigned);
2063  col.setLength(1);
2064  col.setPrimaryKey(false);
2065  col.setNullable(false);
2066  tab.addColumn(col);
2067  tab.setForceVarPart(true);
2068  pDict->dropTable(tab.getName());
2069  if (pDict->createTable(tab))
2070  {
2071  ndbout << pDict->getNdbError() << endl;
2072  return NDBT_FAILED;
2073  }
2074 
2075  const NdbDictionary::Table* pTab = pDict->getTable(tab.getName());
2076  if (pTab == 0)
2077  {
2078  ndbout << pDict->getNdbError() << endl;
2079  return NDBT_FAILED;
2080  }
2081 
2082  {
2083  HugoTransactions trans(* pTab);
2084  if (trans.loadTable(pNdb, rows) != 0)
2085  {
2086  return NDBT_FAILED;
2087  }
2088  }
2089 
2090  res.restartAll2(NdbRestarter::NRRF_NOSTART);
2091  if (res.waitClusterNoStart())
2092  return NDBT_FAILED;
2093  res.startAll();
2094  if (res.waitClusterStarted())
2095  return NDBT_FAILED;
2096 
2097  pNdb->waitUntilReady();
2098 
2099  NdbDictionary::Table newTab = *pTab;
2100  col.setName("ATTR4");
2101  col.setType(NdbDictionary::Column::Varbinary);
2102  col.setLength(25);
2103  col.setPrimaryKey(false);
2104  col.setNullable(true);
2105  col.setDynamic(true);
2106  newTab.addColumn(col);
2107 
2108  if (pDict->alterTable(*pTab, newTab))
2109  {
2110  ndbout << pDict->getNdbError() << endl;
2111  return NDBT_FAILED;
2112  }
2113 
2114  res.restartAll2(NdbRestarter::NRRF_NOSTART | NdbRestarter::NRRF_ABORT);
2115  if (res.waitClusterNoStart())
2116  return NDBT_FAILED;
2117  res.startAll();
2118  if (res.waitClusterStarted())
2119  return NDBT_FAILED;
2120 
2121  pNdb->waitUntilReady();
2122  pDict->dropTable(tab.getName());
2123 
2124  return NDBT_OK;
2125 }
2126 
2127 int
2128 runBug46412(NDBT_Context* ctx, NDBT_Step* step)
2129 {
2130  Uint32 loops = ctx->getNumLoops();
2131  NdbRestarter res;
2132  const Uint32 nodeCount = res.getNumDbNodes();
2133  if(nodeCount < 2)
2134  {
2135  return NDBT_OK;
2136  }
2137 
2138  for (Uint32 l = 0; l<loops; l++)
2139  {
2140 loop:
2141  printf("checking nodegroups of getNextMasterNodeId(): ");
2142  int nodes[256];
2143  bzero(nodes, sizeof(nodes));
2144  nodes[0] = res.getMasterNodeId();
2145  printf("%d ", nodes[0]);
2146  for (Uint32 i = 1; i<nodeCount; i++)
2147  {
2148  nodes[i] = res.getNextMasterNodeId(nodes[i-1]);
2149  printf("%d ", nodes[i]);
2150  }
2151  printf("\n");
2152 
2153  Bitmask<256/32> ng;
2154  int cnt = 0;
2155  int restartnodes[256];
2156 
2157  Uint32 limit = (nodeCount / 2);
2158  for (Uint32 i = 0; i<limit; i++)
2159  {
2160  int tmp = res.getNodeGroup(nodes[i]);
2161  printf("node %d ng: %d", nodes[i], tmp);
2162  if (ng.get(tmp))
2163  {
2164  restartnodes[cnt++] = nodes[i];
2165  ndbout_c(" COLLISION");
2166  limit++;
2167  if (limit > nodeCount)
2168  limit = nodeCount;
2169  }
2170  else
2171  {
2172  ng.set(tmp);
2173  ndbout_c(" OK");
2174  }
2175  }
2176 
2177  if (cnt)
2178  {
2179  printf("restarting nodes: ");
2180  for (int i = 0; i<cnt; i++)
2181  printf("%d ", restartnodes[i]);
2182  printf("\n");
2183  for (int i = 0; i<cnt; i++)
2184  {
2185  res.restartOneDbNode(restartnodes[i], false, true, true);
2186  }
2187  res.waitNodesNoStart(restartnodes, cnt);
2188  res.startNodes(restartnodes, cnt);
2189  if (res.waitClusterStarted())
2190  return NDBT_FAILED;
2191 
2192  goto loop;
2193  }
2194 
2195  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2196  res.dumpStateAllNodes(val2, 2);
2197 
2198  Bitmask<256/32> mask;
2199  for (Uint32 i = 0; i<(nodeCount / 2); i++)
2200  {
2201  int node = nodes[(nodeCount / 2) - (i + 1)];
2202  mask.set(node);
2203  res.insertErrorInNode(node, 7218);
2204  }
2205 
2206  for (Uint32 i = 0; i<nodeCount; i++)
2207  {
2208  int node = nodes[i];
2209  if (mask.get(node))
2210  continue;
2211  res.insertErrorInNode(node, 7220);
2212  }
2213 
2214  int lcp = 7099;
2215  res.dumpStateAllNodes(&lcp, 1);
2216 
2217  res.waitClusterNoStart();
2218  res.startAll();
2219  if (res.waitClusterStarted())
2220  return NDBT_FAILED;
2221  }
2222 
2223  return NDBT_OK;
2224 }
2225 
2226 int
2227 runScanUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step)
2228 {
2229  Ndb* pNdb = GETNDB(step);
2230  HugoTransactions hugoTrans(*ctx->getTab());
2231 
2232  NullOutputStream null;
2233  OutputStream * save[1];
2234  save[0] = g_err.m_out;
2235  g_err.m_out = &null;
2236  while (!ctx->isTestStopped())
2237  {
2238  hugoTrans.scanUpdateRecords(pNdb, 0);
2239  }
2240  g_err.m_out = save[0];
2241  return NDBT_OK;
2242 }
2243 
2244 int
2245 runBug48436(NDBT_Context* ctx, NDBT_Step* step)
2246 {
2247  NdbRestarter res;
2248  Uint32 loops = ctx->getNumLoops();
2249  const Uint32 nodeCount = res.getNumDbNodes();
2250  if(nodeCount < 2)
2251  {
2252  return NDBT_OK;
2253  }
2254 
2255  for (Uint32 l = 0; l<loops; l++)
2256  {
2257  int nodes[2];
2258  nodes[0] = res.getNode(NdbRestarter::NS_RANDOM);
2259  nodes[1] = res.getRandomNodeSameNodeGroup(nodes[0], rand());
2260  int val = 7099;
2261  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2262 
2263  ndbout_c("nodes %u %u", nodes[0], nodes[1]);
2264 
2265  for (Uint32 j = 0; j<5; j++)
2266  {
2267  int c = (rand()) % 11;
2268  ndbout_c("case: %u", c);
2269  switch(c){
2270  case 0:
2271  case 1:
2272  res.dumpStateAllNodes(&val, 1);
2273  case 2:
2274  case 3:
2275  case 4:
2276  case 5:
2277  res.restartOneDbNode(nodes[0], false, true, true);
2278  res.waitNodesNoStart(nodes+0,1);
2279  res.dumpStateOneNode(nodes[0], val2, 2);
2280  res.insertErrorInNode(nodes[0], 5054); // crash during restart
2281  res.startAll();
2282  sleep(3);
2283  res.waitNodesNoStart(nodes+0,1);
2284  res.startAll();
2285  break;
2286  case 6:
2287  res.restartOneDbNode(nodes[0], false, true, true);
2288  res.waitNodesNoStart(nodes+0, 1);
2289  res.startAll();
2290  break;
2291  case 7:
2292  res.dumpStateAllNodes(&val, 1);
2293  case 8:
2294  res.restartOneDbNode(nodes[1], false, true, true);
2295  res.waitNodesNoStart(nodes+1,1);
2296  res.dumpStateOneNode(nodes[1], val2, 2);
2297  res.insertErrorInNode(nodes[1], 5054); // crash during restart
2298  res.startAll();
2299  sleep(3);
2300  res.waitNodesNoStart(nodes+1,1);
2301  res.startAll();
2302  break;
2303  case 9:
2304  res.restartAll(false, true, true);
2305  res.waitClusterNoStart();
2306  res.startAll();
2307  break;
2308  case 10:
2309  {
2310  res.dumpStateAllNodes(val2, 2);
2311  int node = res.getMasterNodeId();
2312  res.insertErrorInNode(node, 7222);
2313  res.waitClusterNoStart();
2314  res.startAll();
2315  break;
2316  }
2317  }
2318  res.waitClusterStarted();
2319  }
2320  res.restartAll(false, true, true);
2321  res.waitClusterNoStart();
2322  res.startAll();
2323  res.waitClusterStarted();
2324  }
2325  ctx->stopTest();
2326 
2327  return NDBT_OK;
2328 }
2329 
2330 int
2331 runBug54611(NDBT_Context* ctx, NDBT_Step* step)
2332 {
2333  NdbRestarter res;
2334  Uint32 loops = ctx->getNumLoops();
2335  Ndb* pNdb = GETNDB(step);
2336  int rows = ctx->getNumRecords();
2337 
2338  HugoTransactions hugoTrans(*ctx->getTab());
2339 
2340  for (Uint32 l = 0; l<loops; l++)
2341  {
2342  int val = DumpStateOrd::DihMinTimeBetweenLCP;
2343  res.dumpStateAllNodes(&val, 1);
2344 
2345  for (Uint32 i = 0; i < 5; i++)
2346  {
2347  hugoTrans.scanUpdateRecords(pNdb, rows);
2348  }
2349 
2350  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2351  res.dumpStateAllNodes(val2, 2);
2352 
2353  int node = res.getMasterNodeId();
2354  res.insertErrorInNode(node, 7222);
2355 
2356  while (hugoTrans.scanUpdateRecords(pNdb, rows) == 0);
2357  res.waitClusterNoStart();
2358 
2359  res.insertErrorInAllNodes(5055);
2360  res.startAll();
2361  res.waitClusterStarted();
2362  pNdb->waitUntilReady();
2363  }
2364 
2365  return NDBT_OK;
2366 }
2367 
2368 int
2369 runBug56961(NDBT_Context* ctx, NDBT_Step* step)
2370 {
2371  NdbRestarter res;
2372  Uint32 loops = ctx->getNumLoops();
2373  Ndb* pNdb = GETNDB(step);
2374  int rows = ctx->getNumRecords();
2375 
2376  int node = res.getNode(NdbRestarter::NS_RANDOM);
2377  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2378  HugoTransactions hugoTrans(*ctx->getTab());
2379 
2380  for (Uint32 l = 0; l<loops; l++)
2381  {
2382  ndbout_c("Waiting for %d to restart (5058)", node);
2383  res.dumpStateOneNode(node, val2, 2);
2384  res.insertErrorInNode(node, 5058);
2385 
2386  hugoTrans.clearTable(pNdb);
2387  hugoTrans.loadTable(pNdb, rows);
2388  while (hugoTrans.scanUpdateRecords(pNdb, rows) == NDBT_OK &&
2389  res.getNodeStatus(node) != NDB_MGM_NODE_STATUS_NOT_STARTED &&
2390  res.getNodeStatus(node) != NDB_MGM_NODE_STATUS_NO_CONTACT);
2391  res.waitNodesNoStart(&node, 1);
2392  res.startNodes(&node, 1);
2393  ndbout_c("Waiting for %d to start", node);
2394  res.waitClusterStarted();
2395 
2396  ndbout_c("Waiting for %d to restart (5059)", node);
2397  res.dumpStateOneNode(node, val2, 2);
2398  res.insertErrorInNode(node, 5059);
2399 
2400  hugoTrans.clearTable(pNdb);
2401  hugoTrans.loadTable(pNdb, rows);
2402  while (hugoTrans.scanUpdateRecords(pNdb, rows) == NDBT_OK &&
2403  res.getNodeStatus(node) != NDB_MGM_NODE_STATUS_NOT_STARTED &&
2404  res.getNodeStatus(node) != NDB_MGM_NODE_STATUS_NO_CONTACT);
2405  res.waitNodesNoStart(&node, 1);
2406  res.startNodes(&node, 1);
2407  ndbout_c("Waiting for %d to start", node);
2408  res.waitClusterStarted();
2409  pNdb->waitUntilReady();
2410  }
2411 
2412  return NDBT_OK;
2413 }
2414 
2415 NDBT_TESTSUITE(testSystemRestart);
2416 TESTCASE("SR1",
2417  "Basic system restart test. Focus on testing restart from REDO log.\n"
2418  "NOTE! Time between lcp's and gcp's should be left at default, \n"
2419  "so that Ndb uses the Redo log when restarting\n"
2420  "1. Load records\n"
2421  "2. Restart cluster and verify records \n"
2422  "3. Update records\n"
2423  "4. Restart cluster and verify records \n"
2424  "5. Delete half of the records \n"
2425  "6. Restart cluster and verify records \n"
2426  "7. Delete all records \n"
2427  "8. Restart cluster and verify records \n"
2428  "9. Insert, update, delete records \n"
2429  "10. Restart cluster and verify records\n"
2430  "11. Insert, update, delete records \n"
2431  "12. Restart cluster with error insert 5020 and verify records\n"){
2432  INITIALIZER(runWaitStarted);
2433  STEP(runSystemRestart1);
2434  FINALIZER(runClearTable);
2435 }
2436 TESTCASE("SR2",
2437  "Basic system restart test. Focus on testing restart from LCP\n"
2438  "NOTE! Time between lcp's is automatically set to it's min value\n"
2439  "so that Ndb uses LCP's when restarting.\n"
2440  "1. Load records\n"
2441  "2. Restart cluster and verify records \n"
2442  "3. Update records\n"
2443  "4. Restart cluster and verify records \n"
2444  "5. Delete half of the records \n"
2445  "6. Restart cluster and verify records \n"
2446  "7. Delete all records \n"
2447  "8. Restart cluster and verify records \n"
2448  "9. Insert, update, delete records \n"
2449  "10. Restart cluster and verify records\n"){
2450  INITIALIZER(runWaitStarted);
2451  STEP(runSystemRestart2);
2452  FINALIZER(runClearTable);
2453 }
2454 TESTCASE("SR_UNDO",
2455  "System restart test. Focus on testing of undologging\n"
2456  "in DBACC and DBTUP.\n"
2457  "This is done by starting a LCP, turn on undologging \n"
2458  "but don't start writing the datapages. This will force all\n"
2459  "operations to be written into the undolog.\n"
2460  "Then write datapages and complete LCP.\n"
2461  "Restart the system\n"){
2462  INITIALIZER(runWaitStarted);
2463  STEP(runSystemRestartTestUndoLog);
2464  FINALIZER(runClearTable);
2465 }
2466 TESTCASE("SR_FULLDB",
2467  "System restart test. Test to restart when DB is full.\n"){
2468  INITIALIZER(runWaitStarted);
2469  STEP(runSystemRestartTestFullDb);
2470  FINALIZER(runClearTable);
2471 }
2472 TESTCASE("SR3",
2473  "System restart test. Focus on testing restart from with\n"
2474  "not all nodes alive when system went down\n"
2475  "* 1. Load data\n"
2476  "* 2. Restart 1 node -nostart\n"
2477  "* 3. Update records\n"
2478  "* 4. Restart cluster and verify records\n"
2479  "* 5. Restart 1 node -nostart\n"
2480  "* 6. Delete half of the records\n"
2481  "* 7. Restart cluster and verify records\n"
2482  "* 8. Restart 1 node -nostart\n"
2483  "* 9. Delete all records\n"
2484  "* 10. Restart cluster and verify records\n"){
2485  INITIALIZER(runWaitStarted);
2486  STEP(runSystemRestart3);
2487  FINALIZER(runClearTable);
2488 }
2489 TESTCASE("SR4",
2490  "System restart test. Focus on testing restart from with\n"
2491  "not all nodes alive when system went down but running LCP at\n"
2492  "high speed so that sometimes a TO is required to start cluster\n"
2493  "* 1. Load data\n"
2494  "* 2. Restart 1 node -nostart\n"
2495  "* 3. Update records\n"
2496  "* 4. Restart cluster and verify records\n"
2497  "* 5. Restart 1 node -nostart\n"
2498  "* 6. Delete half of the records\n"
2499  "* 7. Restart cluster and verify records\n"
2500  "* 8. Restart 1 node -nostart\n"
2501  "* 9. Delete all records\n"
2502  "* 10. Restart cluster and verify records\n"){
2503  INITIALIZER(runWaitStarted);
2504  STEP(runSystemRestart4);
2505  FINALIZER(runClearTable);
2506 }
2507 TESTCASE("SR5",
2508  "As SR4 but making restart aborts\n"
2509  "* 1. Load data\n"
2510  "* 2. Restart 1 node -nostart\n"
2511  "* 3. Update records\n"
2512  "* 4. Restart cluster and verify records\n"
2513  "* 5. Restart 1 node -nostart\n"
2514  "* 6. Delete half of the records\n"
2515  "* 7. Restart cluster and verify records\n"
2516  "* 8. Restart 1 node -nostart\n"
2517  "* 9. Delete all records\n"
2518  "* 10. Restart cluster and verify records\n"){
2519  INITIALIZER(runWaitStarted);
2520  STEP(runSystemRestart5);
2521  FINALIZER(runClearTable);
2522 }
2523 TESTCASE("SR6",
2524  "Perform system restart with some nodes having FS others wo/\n"
2525  "* 1. Load data\n"
2526  "* 2. Restart all node -nostart\n"
2527  "* 3. Restart some nodes -i -nostart\n"
2528  "* 4. Start all nodes verify records\n"){
2529  INITIALIZER(runWaitStarted);
2530  INITIALIZER(runClearTable);
2531  STEP(runSystemRestart6);
2532  FINALIZER(runClearTable);
2533 }
2534 TESTCASE("SR7",
2535  "Perform partition win system restart\n"
2536  "* 1. Load data\n"
2537  "* 2. Restart all node -nostart\n"
2538  "* 3. Start all but one node\n"
2539  "* 4. Verify records\n"
2540  "* 5. Start last node\n"
2541  "* 6. Verify records\n"){
2542  INITIALIZER(runWaitStarted);
2543  INITIALIZER(runClearTable);
2544  STEP(runSystemRestart7);
2545  FINALIZER(runClearTable);
2546 }
2547 TESTCASE("SR8",
2548  "Perform partition win system restart with other nodes delayed\n"
2549  "* 1. Load data\n"
2550  "* 2. Restart all node -nostart\n"
2551  "* 3. Start all but one node\n"
2552  "* 4. Wait for startphase >= 2\n"
2553  "* 5. Start last node\n"
2554  "* 6. Verify records\n"){
2555  INITIALIZER(runWaitStarted);
2556  INITIALIZER(runClearTable);
2557  STEP(runSystemRestart8);
2558  FINALIZER(runClearTable);
2559 }
2560 TESTCASE("SR9",
2561  "Perform partition win system restart with other nodes delayed\n"
2562  "* 1. Start transaction\n"
2563  "* 2. insert (1,1)\n"
2564  "* 3. update (1,2)\n"
2565  "* 4. start lcp\n"
2566  "* 5. commit\n"
2567  "* 6. restart\n"){
2568  INITIALIZER(runWaitStarted);
2569  INITIALIZER(runClearTable);
2570  STEP(runSystemRestart9);
2571  FINALIZER(runClearTable);
2572 }
2573 TESTCASE("Bug18385",
2574  "Perform partition system restart with other nodes with higher GCI"){
2575  INITIALIZER(runWaitStarted);
2576  INITIALIZER(runClearTable);
2577  STEP(runBug18385);
2578  FINALIZER(runClearTable);
2579 }
2580 TESTCASE("Bug21536",
2581  "Perform partition system restart with other nodes with higher GCI"){
2582  INITIALIZER(runWaitStarted);
2583  INITIALIZER(runClearTable);
2584  STEP(runBug21536);
2585  FINALIZER(runClearTable);
2586 }
2587 TESTCASE("Bug24664",
2588  "Check handling of LCP skip/keep")
2589 {
2590  INITIALIZER(runWaitStarted);
2591  INITIALIZER(runClearTable);
2592  STEP(runBug24664);
2593  FINALIZER(runClearTable);
2594 }
2595 TESTCASE("Bug27434",
2596  "")
2597 {
2598  INITIALIZER(runWaitStarted);
2599  STEP(runBug27434);
2600 }
2601 TESTCASE("SR_DD_1", "")
2602 {
2603  TC_PROPERTY("ALL", 1);
2604  INITIALIZER(runWaitStarted);
2605  INITIALIZER(clearOldBackups);
2606  STEP(runStopper);
2607  STEP(runSR_DD_1);
2608  FINALIZER(runClearTable);
2609 }
2610 TESTCASE("SR_DD_1b", "")
2611 {
2612  INITIALIZER(runWaitStarted);
2613  INITIALIZER(clearOldBackups);
2614  STEP(runSR_DD_1);
2615  FINALIZER(runClearTable);
2616 }
2617 TESTCASE("SR_DD_1_LCP", "")
2618 {
2619  TC_PROPERTY("ALL", 1);
2620  TC_PROPERTY("LCP", 1);
2621  INITIALIZER(runWaitStarted);
2622  INITIALIZER(clearOldBackups);
2623  STEP(runStopper);
2624  STEP(runSR_DD_1);
2625  FINALIZER(runClearTable);
2626 }
2627 TESTCASE("SR_DD_1b_LCP", "")
2628 {
2629  TC_PROPERTY("LCP", 1);
2630  INITIALIZER(runWaitStarted);
2631  INITIALIZER(clearOldBackups);
2632  STEP(runSR_DD_1);
2633  FINALIZER(runClearTable);
2634 }
2635 TESTCASE("SR_DD_2", "")
2636 {
2637  TC_PROPERTY("ALL", 1);
2638  INITIALIZER(runWaitStarted);
2639  INITIALIZER(clearOldBackups);
2640  STEP(runStopper);
2641  STEP(runSR_DD_2);
2642  FINALIZER(runClearTable);
2643 }
2644 TESTCASE("SR_DD_2b", "")
2645 {
2646  INITIALIZER(runWaitStarted);
2647  INITIALIZER(clearOldBackups);
2648  STEP(runSR_DD_2);
2649  FINALIZER(runClearTable);
2650 }
2651 TESTCASE("SR_DD_2_LCP", "")
2652 {
2653  TC_PROPERTY("ALL", 1);
2654  TC_PROPERTY("LCP", 1);
2655  INITIALIZER(runWaitStarted);
2656  INITIALIZER(clearOldBackups);
2657  STEP(runStopper);
2658  STEP(runSR_DD_2);
2659  FINALIZER(runClearTable);
2660 }
2661 TESTCASE("SR_DD_2b_LCP", "")
2662 {
2663  TC_PROPERTY("LCP", 1);
2664  INITIALIZER(runWaitStarted);
2665  INITIALIZER(clearOldBackups);
2666  STEP(runSR_DD_2);
2667  FINALIZER(runClearTable);
2668 }
2669 TESTCASE("SR_DD_3", "")
2670 {
2671  TC_PROPERTY("ALL", 1);
2672  INITIALIZER(runWaitStarted);
2673  INITIALIZER(clearOldBackups);
2674  STEP(runStopper);
2675  STEP(runSR_DD_3);
2676  FINALIZER(runClearTable);
2677 }
2678 TESTCASE("SR_DD_3b", "")
2679 {
2680  INITIALIZER(runWaitStarted);
2681  INITIALIZER(clearOldBackups);
2682  STEP(runSR_DD_3);
2683  FINALIZER(runClearTable);
2684 }
2685 TESTCASE("SR_DD_3_LCP", "")
2686 {
2687  TC_PROPERTY("ALL", 1);
2688  TC_PROPERTY("LCP", 1);
2689  INITIALIZER(runWaitStarted);
2690  INITIALIZER(clearOldBackups);
2691  STEP(runStopper);
2692  STEP(runSR_DD_3);
2693  FINALIZER(runClearTable);
2694 }
2695 TESTCASE("SR_DD_3b_LCP", "")
2696 {
2697  TC_PROPERTY("LCP", 1);
2698  INITIALIZER(runWaitStarted);
2699  INITIALIZER(clearOldBackups);
2700  STEP(runSR_DD_3);
2701  FINALIZER(runClearTable);
2702 }
2703 TESTCASE("Bug29167", "")
2704 {
2705  INITIALIZER(runWaitStarted);
2706  STEP(runBug29167);
2707 }
2708 TESTCASE("Bug28770",
2709  "Check readTableFile1 fails, readTableFile2 succeeds\n"
2710  "1. Restart all node -nostart\n"
2711  "2. Insert error 6100 into all nodes\n"
2712  "3. Start all nodes\n"
2713  "4. Ensure cluster start\n"
2714  "5. Read and verify reocrds\n"
2715  "6. Repeat until looping is completed\n"){
2716  INITIALIZER(runWaitStarted);
2717  INITIALIZER(runClearTable);
2718  STEP(runBug28770);
2719  FINALIZER(runClearTable);
2720 }
2721 TESTCASE("Bug22696", "")
2722 {
2723  INITIALIZER(runWaitStarted);
2724  INITIALIZER(runLoadTable);
2725  INITIALIZER(runBug22696);
2726  FINALIZER(runClearTable);
2727 }
2728 TESTCASE("to", "Take-over during SR")
2729 {
2730  INITIALIZER(runWaitStarted);
2731  INITIALIZER(runLoadTable);
2732  INITIALIZER(runTO);
2733  FINALIZER(runClearTable);
2734 }
2735 TESTCASE("basic", "")
2736 {
2737  INITIALIZER(runWaitStarted);
2738  INITIALIZER(runCreateAllTables);
2739  STEP(runBasic);
2740  FINALIZER(runDropAllTables);
2741 }
2742 TESTCASE("Bug41915", "")
2743 {
2744  TC_PROPERTY("ALL", 1);
2745  TC_PROPERTY("ERROR", 5053);
2746  TC_PROPERTY("ROWS", 30);
2747  INITIALIZER(runWaitStarted);
2748  STEP(runStopper);
2749  STEP(runSR_DD_2);
2750  FINALIZER(runClearTable);
2751 }
2752 TESTCASE("Bug45154", "")
2753 {
2754  INITIALIZER(runBug45154);
2755 }
2756 TESTCASE("Bug46651", "")
2757 {
2758  INITIALIZER(runBug46651);
2759 }
2760 TESTCASE("Bug46412", "")
2761 {
2762  INITIALIZER(runBug46412);
2763 }
2764 TESTCASE("Bug48436", "")
2765 {
2766  INITIALIZER(runLoadTable);
2767  STEP(runBug48436);
2768  STEP(runScanUpdateUntilStopped);
2769 }
2770 TESTCASE("Bug54611", "")
2771 {
2772  INITIALIZER(runLoadTable);
2773  INITIALIZER(runBug54611);
2774 }
2775 TESTCASE("Bug56961", "")
2776 {
2777  INITIALIZER(runLoadTable);
2778  INITIALIZER(runBug56961);
2779 }
2780 NDBT_TESTSUITE_END(testSystemRestart);
2781 
2782 int main(int argc, const char** argv){
2783  ndb_init();
2784  NDBT_TESTSUITE_INSTANCE(testSystemRestart);
2785  return testSystemRestart.execute(argc, argv);
2786 }
2787 
2788