MySQL 5.6.14 Source Code Document
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
testNodeRestart.cpp
1 /*
2  Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
3 
4  This program is free software; you can redistribute it and/or modify
5  it under the terms of the GNU General Public License as published by
6  the Free Software Foundation; version 2 of the License.
7 
8  This program is distributed in the hope that it will be useful,
9  but WITHOUT ANY WARRANTY; without even the implied warranty of
10  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  GNU General Public License for more details.
12 
13  You should have received a copy of the GNU General Public License
14  along with this program; if not, write to the Free Software
15  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
16 */
17 
18 #include <NDBT.hpp>
19 #include <NDBT_Test.hpp>
20 #include <HugoTransactions.hpp>
21 #include <UtilTransactions.hpp>
22 #include <NdbRestarter.hpp>
23 #include <NdbRestarts.hpp>
24 #include <Vector.hpp>
25 #include <signaldata/DumpStateOrd.hpp>
26 #include <Bitmask.hpp>
27 #include <RefConvert.hpp>
28 #include <NdbEnv.h>
29 
30 int runLoadTable(NDBT_Context* ctx, NDBT_Step* step){
31 
32  int records = ctx->getNumRecords();
33  HugoTransactions hugoTrans(*ctx->getTab());
34  if (hugoTrans.loadTable(GETNDB(step), records) != 0){
35  return NDBT_FAILED;
36  }
37  return NDBT_OK;
38 }
39 
40 int runFillTable(NDBT_Context* ctx, NDBT_Step* step){
41 
42  HugoTransactions hugoTrans(*ctx->getTab());
43  if (hugoTrans.fillTable(GETNDB(step)) != 0){
44  return NDBT_FAILED;
45  }
46  return NDBT_OK;
47 }
48 
49 int runInsertUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
50  int result = NDBT_OK;
51  int records = ctx->getNumRecords();
52  int i = 0;
53  HugoTransactions hugoTrans(*ctx->getTab());
54  while (ctx->isTestStopped() == false) {
55  g_info << i << ": ";
56  if (hugoTrans.loadTable(GETNDB(step), records) != 0){
57  return NDBT_FAILED;
58  }
59  i++;
60  }
61  return result;
62 }
63 
64 int runClearTable(NDBT_Context* ctx, NDBT_Step* step){
65  int records = ctx->getNumRecords();
66 
67  UtilTransactions utilTrans(*ctx->getTab());
68  if (utilTrans.clearTable(GETNDB(step), records) != 0){
69  return NDBT_FAILED;
70  }
71  return NDBT_OK;
72 }
73 
74 int runClearTableUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
75  int records = ctx->getNumRecords();
76  int i = 0;
77 
78  UtilTransactions utilTrans(*ctx->getTab());
79  while (ctx->isTestStopped() == false) {
80  g_info << i << ": ";
81  if (utilTrans.clearTable(GETNDB(step), records) != 0){
82  return NDBT_FAILED;
83  }
84  i++;
85  }
86  return NDBT_OK;
87 }
88 
89 int runScanReadUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
90  int result = NDBT_OK;
91  int records = ctx->getNumRecords();
92  int i = 0;
93  HugoTransactions hugoTrans(*ctx->getTab());
94  while (ctx->isTestStopped() == false) {
95  g_info << i << ": ";
96  if (hugoTrans.scanReadRecords(GETNDB(step), records) != 0){
97  return NDBT_FAILED;
98  }
99  i++;
100  }
101  return result;
102 }
103 
104 int runPkReadUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
105  int result = NDBT_OK;
106  int records = ctx->getNumRecords();
107  NdbOperation::LockMode lm =
108  (NdbOperation::LockMode)ctx->getProperty("ReadLockMode",
109  (Uint32)NdbOperation::LM_Read);
110  int i = 0;
111  HugoTransactions hugoTrans(*ctx->getTab());
112  while (ctx->isTestStopped() == false) {
113  g_info << i << ": ";
114  int rows = (rand()%records)+1;
115  int batch = (rand()%rows)+1;
116  if (hugoTrans.pkReadRecords(GETNDB(step), rows, batch, lm) != 0){
117  return NDBT_FAILED;
118  }
119  i++;
120  }
121  return result;
122 }
123 
124 int runPkUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
125  int result = NDBT_OK;
126  int records = ctx->getNumRecords();
127  int multiop = ctx->getProperty("MULTI_OP", 1);
128  Ndb* pNdb = GETNDB(step);
129  int i = 0;
130 
131  HugoOperations hugoOps(*ctx->getTab());
132  while (ctx->isTestStopped() == false)
133  {
134  g_info << i << ": ";
135  int batch = (rand()%records)+1;
136  int row = rand() % records;
137 
138  if (batch > 25)
139  batch = 25;
140 
141  if(row + batch > records)
142  batch = records - row;
143 
144  if(hugoOps.startTransaction(pNdb) != 0)
145  goto err;
146 
147  if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
148  goto err;
149 
150  for (int j = 1; j<multiop; j++)
151  {
152  if(hugoOps.execute_NoCommit(pNdb) != 0)
153  goto err;
154 
155  if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
156  goto err;
157  }
158 
159  if(hugoOps.execute_Commit(pNdb) != 0)
160  goto err;
161 
162  hugoOps.closeTransaction(pNdb);
163 
164  continue;
165 
166 err:
167  NdbConnection* pCon = hugoOps.getTransaction();
168  if(pCon == 0)
169  continue;
170  NdbError error = pCon->getNdbError();
171  hugoOps.closeTransaction(pNdb);
172  if (error.status == NdbError::TemporaryError){
173  NdbSleep_MilliSleep(50);
174  continue;
175  }
176  return NDBT_FAILED;
177 
178  i++;
179  }
180  return result;
181 }
182 
183 int runPkReadPkUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step)
184 {
185  int records = ctx->getNumRecords();
186  Ndb* pNdb = GETNDB(step);
187  int i = 0;
188  HugoOperations hugoOps(*ctx->getTab());
189  while (ctx->isTestStopped() == false) {
190  g_info << i++ << ": ";
191  int rows = (rand()%records)+1;
192  int batch = (rand()%rows)+1;
193  int row = (records - rows) ? rand() % (records - rows) : 0;
194 
195  int j,k;
196  for(j = 0; j<rows; j += batch)
197  {
198  k = batch;
199  if(j+k > rows)
200  k = rows - j;
201 
202  if(hugoOps.startTransaction(pNdb) != 0)
203  goto err;
204 
205  if(hugoOps.pkReadRecord(pNdb, row+j, k, NdbOperation::LM_Exclusive) != 0)
206  goto err;
207 
208  if(hugoOps.execute_NoCommit(pNdb) != 0)
209  goto err;
210 
211  if(hugoOps.pkUpdateRecord(pNdb, row+j, k, rand()) != 0)
212  goto err;
213 
214  if(hugoOps.execute_Commit(pNdb) != 0)
215  goto err;
216 
217  if(hugoOps.closeTransaction(pNdb) != 0)
218  return NDBT_FAILED;
219  }
220 
221  continue;
222 err:
223  NdbConnection* pCon = hugoOps.getTransaction();
224  if(pCon == 0)
225  continue;
226  NdbError error = pCon->getNdbError();
227  hugoOps.closeTransaction(pNdb);
228  if (error.status == NdbError::TemporaryError){
229  NdbSleep_MilliSleep(50);
230  continue;
231  }
232  return NDBT_FAILED;
233  }
234  return NDBT_OK;
235 }
236 
237 int runPkReadPkUpdatePkUnlockUntilStopped(NDBT_Context* ctx, NDBT_Step* step)
238 {
239  int records = ctx->getNumRecords();
240  Ndb* pNdb = GETNDB(step);
241  int i = 0;
242  HugoOperations hugoOps(*ctx->getTab());
243  while (ctx->isTestStopped() == false) {
244  g_info << i++ << ": ";
245  int rows = (rand()%records)+1;
246  int batch = (rand()%rows)+1;
247  int row = (records - rows) ? rand() % (records - rows) : 0;
248 
249  int j,k;
250  for(j = 0; j<rows; j += batch)
251  {
252  k = batch;
253  if(j+k > rows)
254  k = rows - j;
255 
256  Vector<const NdbLockHandle*> lockHandles;
257 
258  if(hugoOps.startTransaction(pNdb) != 0)
259  goto err;
260 
261  if(hugoOps.pkReadRecordLockHandle(pNdb, lockHandles, row+j, k, NdbOperation::LM_Exclusive) != 0)
262  goto err;
263 
264  if(hugoOps.execute_NoCommit(pNdb) != 0)
265  goto err;
266 
267  if(hugoOps.pkUpdateRecord(pNdb, row+j, k, rand()) != 0)
268  goto err;
269 
270  if(hugoOps.execute_NoCommit(pNdb) != 0)
271  goto err;
272 
273  if(hugoOps.pkUnlockRecord(pNdb, lockHandles) != 0)
274  goto err;
275 
276  if(hugoOps.execute_Commit(pNdb) != 0)
277  goto err;
278 
279  if(hugoOps.closeTransaction(pNdb) != 0)
280  return NDBT_FAILED;
281  }
282 
283  continue;
284 err:
285  NdbConnection* pCon = hugoOps.getTransaction();
286  if(pCon == 0)
287  continue;
288  NdbError error = pCon->getNdbError();
289  hugoOps.closeTransaction(pNdb);
290  if (error.status == NdbError::TemporaryError){
291  NdbSleep_MilliSleep(50);
292  continue;
293  }
294  return NDBT_FAILED;
295  }
296  return NDBT_OK;
297 }
298 
299 int runDeleteInsertUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
300  int result = NDBT_OK;
301  int records = ctx->getNumRecords();
302  int i = 0;
303  HugoTransactions hugoTrans(*ctx->getTab());
304  UtilTransactions utilTrans(*ctx->getTab());
305  while (ctx->isTestStopped() == false)
306  {
307  g_info << i << ": ";
308  if (utilTrans.clearTable(GETNDB(step), records) != 0){
309  result = NDBT_FAILED;
310  break;
311  }
312  if (hugoTrans.loadTable(GETNDB(step), records, 1) != 0){
313  result = NDBT_FAILED;
314  break;
315  }
316  i++;
317  }
318 
319  return result;
320 }
321 
322 int runScanUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
323  int result = NDBT_OK;
324  int records = ctx->getNumRecords();
325  int parallelism = ctx->getProperty("Parallelism", 1);
326  int abort = ctx->getProperty("AbortProb", (Uint32)0);
327  int check = ctx->getProperty("ScanUpdateNoRowCountCheck", (Uint32)0);
328 
329  if (check)
330  records = 0;
331 
332  int i = 0;
333  HugoTransactions hugoTrans(*ctx->getTab());
334  while (ctx->isTestStopped() == false) {
335  g_info << i << ": ";
336  if (hugoTrans.scanUpdateRecords(GETNDB(step), records, abort,
337  parallelism) == NDBT_FAILED){
338  return NDBT_FAILED;
339  }
340  i++;
341  }
342  return result;
343 }
344 
345 int runScanReadVerify(NDBT_Context* ctx, NDBT_Step* step){
346  int records = ctx->getNumRecords();
347  HugoTransactions hugoTrans(*ctx->getTab());
348 
349  if (hugoTrans.scanReadRecords(GETNDB(step), records, 0, 64) != 0){
350  return NDBT_FAILED;
351  }
352  return NDBT_OK;
353 }
354 
355 int runRestarter(NDBT_Context* ctx, NDBT_Step* step){
356  int result = NDBT_OK;
357  int loops = ctx->getNumLoops();
358  int sync_threads = ctx->getProperty("SyncThreads", (unsigned)0);
359  int sleep0 = ctx->getProperty("Sleep0", (unsigned)0);
360  int sleep1 = ctx->getProperty("Sleep1", (unsigned)0);
361  int randnode = ctx->getProperty("RandNode", (unsigned)0);
362  NdbRestarter restarter;
363  int i = 0;
364  int lastId = 0;
365 
366  if (restarter.getNumDbNodes() < 2){
367  ctx->stopTest();
368  return NDBT_OK;
369  }
370 
371  if(restarter.waitClusterStarted() != 0){
372  g_err << "Cluster failed to start" << endl;
373  return NDBT_FAILED;
374  }
375 
376  loops *= (restarter.getNumDbNodes() > 2 ? 2 : restarter.getNumDbNodes());
377  if (loops < restarter.getNumDbNodes())
378  loops = restarter.getNumDbNodes();
379 
380  while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
381 
382  int id = lastId % restarter.getNumDbNodes();
383  if (randnode == 1)
384  {
385  id = rand() % restarter.getNumDbNodes();
386  }
387  int nodeId = restarter.getDbNodeId(id);
388  ndbout << "Restart node " << nodeId << endl;
389  if(restarter.restartOneDbNode(nodeId, false, true, true) != 0){
390  g_err << "Failed to restartNextDbNode" << endl;
391  result = NDBT_FAILED;
392  break;
393  }
394 
395  if (restarter.waitNodesNoStart(&nodeId, 1))
396  {
397  g_err << "Failed to waitNodesNoStart" << endl;
398  result = NDBT_FAILED;
399  break;
400  }
401 
402  if (sleep1)
403  NdbSleep_MilliSleep(sleep1);
404 
405  if (restarter.startNodes(&nodeId, 1))
406  {
407  g_err << "Failed to start node" << endl;
408  result = NDBT_FAILED;
409  break;
410  }
411 
412  if(restarter.waitClusterStarted() != 0){
413  g_err << "Cluster failed to start" << endl;
414  result = NDBT_FAILED;
415  break;
416  }
417 
418  if (sleep0)
419  NdbSleep_MilliSleep(sleep0);
420 
421  ctx->sync_up_and_wait("PauseThreads", sync_threads);
422 
423  lastId++;
424  i++;
425  }
426 
427  ctx->stopTest();
428 
429  return result;
430 }
431 
432 int runCheckAllNodesStarted(NDBT_Context* ctx, NDBT_Step* step){
433  NdbRestarter restarter;
434 
435  if(restarter.waitClusterStarted(1) != 0){
436  g_err << "All nodes was not started " << endl;
437  return NDBT_FAILED;
438  }
439 
440  return NDBT_OK;
441 }
442 
443 
444 
445 int runRestarts(NDBT_Context* ctx, NDBT_Step* step){
446  int result = NDBT_OK;
447  int loops = ctx->getNumLoops();
448  NDBT_TestCase* pCase = ctx->getCase();
449  NdbRestarts restarts;
450  int i = 0;
451  int timeout = 240;
452 
453  while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
454 
455  if(restarts.executeRestart(ctx, pCase->getName(), timeout) != 0){
456  g_err << "Failed to executeRestart(" <<pCase->getName() <<")" << endl;
457  result = NDBT_FAILED;
458  break;
459  }
460  i++;
461  }
462  ctx->stopTest();
463  return result;
464 }
465 
466 int runDirtyRead(NDBT_Context* ctx, NDBT_Step* step){
467  int result = NDBT_OK;
468  int loops = ctx->getNumLoops();
469  int records = ctx->getNumRecords();
470  NdbRestarter restarter;
471  HugoOperations hugoOps(*ctx->getTab());
472  Ndb* pNdb = GETNDB(step);
473 
474  int i = 0;
475  while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
476  g_info << i << ": ";
477 
478  int id = i % restarter.getNumDbNodes();
479  int nodeId = restarter.getDbNodeId(id);
480  ndbout << "Restart node " << nodeId << endl;
481  restarter.insertErrorInNode(nodeId, 5041);
482  restarter.insertErrorInAllNodes(8048 + (i & 1));
483 
484  for(int j = 0; j<records; j++){
485  if(hugoOps.startTransaction(pNdb) != 0)
486  return NDBT_FAILED;
487 
488  if(hugoOps.pkReadRecord(pNdb, j, 1, NdbOperation::LM_CommittedRead) != 0)
489  goto err;
490 
491  int res;
492  if((res = hugoOps.execute_Commit(pNdb)) == 4119)
493  goto done;
494 
495  if(res != 0)
496  goto err;
497 
498  if(hugoOps.closeTransaction(pNdb) != 0)
499  return NDBT_FAILED;
500  }
501 done:
502  if(hugoOps.closeTransaction(pNdb) != 0)
503  return NDBT_FAILED;
504 
505  i++;
506  restarter.waitClusterStarted(60) ;
507  }
508  return result;
509 err:
510  hugoOps.closeTransaction(pNdb);
511  return NDBT_FAILED;
512 }
513 
514 int runLateCommit(NDBT_Context* ctx, NDBT_Step* step)
515 {
516  int result = NDBT_OK;
517  int loops = ctx->getNumLoops();
518  NdbRestarter restarter;
519  HugoOperations hugoOps(*ctx->getTab());
520  Ndb* pNdb = GETNDB(step);
521 
522  int i = 0;
523  while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
524  g_info << i << ": ";
525 
526  if(hugoOps.startTransaction(pNdb) != 0)
527  return NDBT_FAILED;
528 
529  if(hugoOps.pkUpdateRecord(pNdb, 1, 128) != 0)
530  return NDBT_FAILED;
531 
532  if(hugoOps.execute_NoCommit(pNdb) != 0)
533  return NDBT_FAILED;
534 
535  Uint32 transNode= hugoOps.getTransaction()->getConnectedNodeId();
536  int id = i % restarter.getNumDbNodes();
537  int nodeId;
538  while((nodeId = restarter.getDbNodeId(id)) == (int)transNode)
539  id = (id + 1) % restarter.getNumDbNodes();
540 
541  ndbout << "Restart node " << nodeId << endl;
542 
543  restarter.restartOneDbNode(nodeId, false, true, true);
547 
548  restarter.waitNodesNoStart(&nodeId, 1);
549 
550  int res;
551  if(i & 1)
552  res= hugoOps.execute_Commit(pNdb);
553  else
554  res= hugoOps.execute_Rollback(pNdb);
555 
556  ndbout_c("res= %d", res);
557 
558  hugoOps.closeTransaction(pNdb);
559 
560  restarter.startNodes(&nodeId, 1);
561  restarter.waitNodesStarted(&nodeId, 1);
562 
563  if(i & 1)
564  {
565  if(res != 286)
566  return NDBT_FAILED;
567  }
568  else
569  {
570  if(res != 0)
571  return NDBT_FAILED;
572  }
573  i++;
574  }
575 
576  return NDBT_OK;
577 }
578 
579 int runBug15587(NDBT_Context* ctx, NDBT_Step* step)
580 {
581  NdbRestarter restarter;
582 
583  Uint32 tableId = ctx->getTab()->getTableId();
584  int dump[2] = { DumpStateOrd::LqhErrorInsert5042, 0 };
585  dump[1] = tableId;
586 
587  int nodeId = restarter.getDbNodeId(1);
588 
589  ndbout << "Restart node " << nodeId << endl;
590 
591  if (restarter.restartOneDbNode(nodeId, false, true, true))
595  return NDBT_FAILED;
596 
597  if (restarter.waitNodesNoStart(&nodeId, 1))
598  return NDBT_FAILED;
599 
600  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
601 
602  if (restarter.dumpStateOneNode(nodeId, val2, 2))
603  return NDBT_FAILED;
604 
605  if (restarter.dumpStateOneNode(nodeId, dump, 2))
606  return NDBT_FAILED;
607 
608  if (restarter.startNodes(&nodeId, 1))
609  return NDBT_FAILED;
610 
611  restarter.waitNodesStartPhase(&nodeId, 1, 3);
612 
613  if (restarter.waitNodesNoStart(&nodeId, 1))
614  return NDBT_FAILED;
615 
616  if (restarter.dumpStateOneNode(nodeId, val2, 1))
617  return NDBT_FAILED;
618 
619  if (restarter.startNodes(&nodeId, 1))
620  return NDBT_FAILED;
621 
622  if (restarter.waitNodesStarted(&nodeId, 1))
623  return NDBT_FAILED;
624 
625  ctx->stopTest();
626  return NDBT_OK;
627 }
628 
629 int runBug15632(NDBT_Context* ctx, NDBT_Step* step)
630 {
631  NdbRestarter restarter;
632 
633  int nodeId = restarter.getDbNodeId(1);
634 
635  ndbout << "Restart node " << nodeId << endl;
636 
637  if (restarter.restartOneDbNode(nodeId, false, true, true))
641  return NDBT_FAILED;
642 
643  if (restarter.waitNodesNoStart(&nodeId, 1))
644  return NDBT_FAILED;
645 
646  if (restarter.insertErrorInNode(nodeId, 7165))
647  return NDBT_FAILED;
648 
649  if (restarter.startNodes(&nodeId, 1))
650  return NDBT_FAILED;
651 
652  if (restarter.waitNodesStarted(&nodeId, 1))
653  return NDBT_FAILED;
654 
655  if (restarter.restartOneDbNode(nodeId, false, true, true))
659  return NDBT_FAILED;
660 
661  if (restarter.waitNodesNoStart(&nodeId, 1))
662  return NDBT_FAILED;
663 
664  if (restarter.insertErrorInNode(nodeId, 7171))
665  return NDBT_FAILED;
666 
667  if (restarter.startNodes(&nodeId, 1))
668  return NDBT_FAILED;
669 
670  if (restarter.waitNodesStarted(&nodeId, 1))
671  return NDBT_FAILED;
672 
673  ctx->stopTest();
674  return NDBT_OK;
675 }
676 
677 int runBug15685(NDBT_Context* ctx, NDBT_Step* step){
678 
679  Ndb* pNdb = GETNDB(step);
680  HugoOperations hugoOps(*ctx->getTab());
681  NdbRestarter restarter;
682 
683  HugoTransactions hugoTrans(*ctx->getTab());
684  if (hugoTrans.loadTable(GETNDB(step), 10) != 0){
685  return NDBT_FAILED;
686  }
687 
688  if(hugoOps.startTransaction(pNdb) != 0)
689  goto err;
690 
691  if(hugoOps.pkUpdateRecord(pNdb, 0, 1, rand()) != 0)
692  goto err;
693 
694  if(hugoOps.execute_NoCommit(pNdb) != 0)
695  goto err;
696 
697  if (restarter.insertErrorInAllNodes(5100))
698  return NDBT_FAILED;
699 
700  hugoOps.execute_Rollback(pNdb);
701 
702  if (restarter.waitClusterStarted() != 0)
703  goto err;
704 
705  if (restarter.insertErrorInAllNodes(0))
706  return NDBT_FAILED;
707 
708  ctx->stopTest();
709  return NDBT_OK;
710 
711 err:
712  ctx->stopTest();
713  return NDBT_FAILED;
714 }
715 
716 int
717 runBug16772(NDBT_Context* ctx, NDBT_Step* step){
718 
719  NdbRestarter restarter;
720  if (restarter.getNumDbNodes() < 2)
721  {
722  ctx->stopTest();
723  return NDBT_OK;
724  }
725 
726  int aliveNodeId = restarter.getRandomNotMasterNodeId(rand());
727  int deadNodeId = aliveNodeId;
728  while (deadNodeId == aliveNodeId)
729  deadNodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
730 
731  if (restarter.insertErrorInNode(aliveNodeId, 930))
732  return NDBT_FAILED;
733 
734  if (restarter.restartOneDbNode(deadNodeId, false, true, true))
738  return NDBT_FAILED;
739 
740  if (restarter.waitNodesNoStart(&deadNodeId, 1))
741  return NDBT_FAILED;
742 
743  if (restarter.startNodes(&deadNodeId, 1))
744  return NDBT_FAILED;
745 
746  // It should now be hanging since we throw away NDB_FAILCONF
747  int ret = restarter.waitNodesStartPhase(&deadNodeId, 1, 3, 10);
748  // So this should fail...i.e it should not reach startphase 3
749 
750  // Now send a NDB_FAILCONF for deadNo
751  int dump[] = { 7020, 323, 252, 0 };
752  dump[3] = deadNodeId;
753  if (restarter.dumpStateOneNode(aliveNodeId, dump, 4))
754  return NDBT_FAILED;
755 
756  if (restarter.waitNodesStarted(&deadNodeId, 1))
757  return NDBT_FAILED;
758 
759  return ret ? NDBT_OK : NDBT_FAILED;
760 }
761 
762 int
763 runBug18414(NDBT_Context* ctx, NDBT_Step* step){
764 
765  NdbRestarter restarter;
766  if (restarter.getNumDbNodes() < 2)
767  {
768  ctx->stopTest();
769  return NDBT_OK;
770  }
771 
772  Ndb* pNdb = GETNDB(step);
773  HugoOperations hugoOps(*ctx->getTab());
774  HugoTransactions hugoTrans(*ctx->getTab());
775  int loop = 0;
776  do
777  {
778  if(hugoOps.startTransaction(pNdb) != 0)
779  goto err;
780 
781  if(hugoOps.pkUpdateRecord(pNdb, 0, 128, rand()) != 0)
782  goto err;
783 
784  if(hugoOps.execute_NoCommit(pNdb) != 0)
785  goto err;
786 
787  int node1 = hugoOps.getTransaction()->getConnectedNodeId();
788  int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
789 
790  if (node1 == -1 || node2 == -1)
791  break;
792 
793  if (loop & 1)
794  {
795  if (restarter.insertErrorInNode(node1, 8080))
796  goto err;
797  }
798 
799  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
800  if (restarter.dumpStateOneNode(node2, val2, 2))
801  goto err;
802 
803  if (restarter.insertErrorInNode(node2, 5003))
804  goto err;
805  hugoOps.execute_Rollback(pNdb);
807 
808  if (restarter.waitNodesNoStart(&node2, 1) != 0)
809  goto err;
810 
811  if (restarter.insertErrorInAllNodes(0))
812  goto err;
813 
814  if (restarter.startNodes(&node2, 1) != 0)
815  goto err;
816 
817  if (restarter.waitClusterStarted() != 0)
818  goto err;
819 
820  if (hugoTrans.scanUpdateRecords(pNdb, 128) != 0)
821  goto err;
822 
823  hugoOps.closeTransaction(pNdb);
824 
825  } while(++loop < 5);
826 
827  return NDBT_OK;
828 
829 err:
830  hugoOps.closeTransaction(pNdb);
831  return NDBT_FAILED;
832 }
833 
834 int
835 runBug18612(NDBT_Context* ctx, NDBT_Step* step){
836 
837  // Assume two replicas
838  NdbRestarter restarter;
839  if (restarter.getNumDbNodes() < 2)
840  {
841  ctx->stopTest();
842  return NDBT_OK;
843  }
844 
845  Uint32 cnt = restarter.getNumDbNodes();
846 
847  for(int loop = 0; loop < ctx->getNumLoops(); loop++)
848  {
849  int partition0[256];
850  int partition1[256];
851  memset(partition0, 0, sizeof(partition0));
852  memset(partition1, 0, sizeof(partition1));
853  Bitmask<4> nodesmask;
854 
855  Uint32 node1 = restarter.getDbNodeId(rand()%cnt);
856  for (Uint32 i = 0; i<cnt/2; i++)
857  {
858  do {
859  int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand());
860  if (tmp == -1)
861  {
862  ctx->stopTest();
863  return NDBT_OK;
864  }
865  node1 = tmp;
866  } while(nodesmask.get(node1));
867 
868  partition0[i] = node1;
869  partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand());
870 
871  ndbout_c("nodes %d %d", node1, partition1[i]);
872 
873  assert(!nodesmask.get(node1));
874  assert(!nodesmask.get(partition1[i]));
875  nodesmask.set(node1);
876  nodesmask.set(partition1[i]);
877  }
878 
879  ndbout_c("done");
880 
881  int dump[255];
882  dump[0] = DumpStateOrd::NdbcntrStopNodes;
883  memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
884 
885  Uint32 master = restarter.getMasterNodeId();
886 
887  if (restarter.dumpStateOneNode(master, dump, 1+cnt/2))
888  return NDBT_FAILED;
889 
890  if (restarter.waitNodesNoStart(partition0, cnt/2))
891  return NDBT_FAILED;
892 
893  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
894 
895  if (restarter.dumpStateAllNodes(val2, 2))
896  return NDBT_FAILED;
897 
898  if (restarter.insertErrorInAllNodes(932))
899  return NDBT_FAILED;
900 
901  dump[0] = 9000;
902  memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
903  for (Uint32 i = 0; i<cnt/2; i++)
904  if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2))
905  return NDBT_FAILED;
906 
907  dump[0] = 9000;
908  memcpy(dump + 1, partition1, sizeof(int)*cnt/2);
909  for (Uint32 i = 0; i<cnt/2; i++)
910  if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2))
911  return NDBT_FAILED;
912 
913  if (restarter.startNodes(partition0, cnt/2))
914  return NDBT_FAILED;
915 
916  if (restarter.waitNodesStartPhase(partition0, cnt/2, 2))
917  return NDBT_FAILED;
918 
919  dump[0] = 9001;
920  for (Uint32 i = 0; i<cnt/2; i++)
921  if (restarter.dumpStateAllNodes(dump, 2))
922  return NDBT_FAILED;
923 
924  if (restarter.waitNodesNoStart(partition0, cnt/2))
925  return NDBT_FAILED;
926 
927  for (Uint32 i = 0; i<cnt/2; i++)
928  if (restarter.restartOneDbNode(partition0[i], true, true, true))
929  return NDBT_FAILED;
930 
931  if (restarter.waitNodesNoStart(partition0, cnt/2))
932  return NDBT_FAILED;
933 
934  if (restarter.startAll())
935  return NDBT_FAILED;
936 
937  if (restarter.waitClusterStarted())
938  return NDBT_FAILED;
939  }
940  return NDBT_OK;
941 }
942 
943 int
944 runBug18612SR(NDBT_Context* ctx, NDBT_Step* step){
945 
946  // Assume two replicas
947  NdbRestarter restarter;
948  if (restarter.getNumDbNodes() < 2)
949  {
950  ctx->stopTest();
951  return NDBT_OK;
952  }
953 
954  Uint32 cnt = restarter.getNumDbNodes();
955 
956  for(int loop = 0; loop < ctx->getNumLoops(); loop++)
957  {
958  int partition0[256];
959  int partition1[256];
960  memset(partition0, 0, sizeof(partition0));
961  memset(partition1, 0, sizeof(partition1));
962  Bitmask<4> nodesmask;
963 
964  Uint32 node1 = restarter.getDbNodeId(rand()%cnt);
965  for (Uint32 i = 0; i<cnt/2; i++)
966  {
967  do {
968  int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand());
969  if (tmp == -1)
970  break;
971  node1 = tmp;
972  } while(nodesmask.get(node1));
973 
974  partition0[i] = node1;
975  partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand());
976 
977  ndbout_c("nodes %d %d", node1, partition1[i]);
978 
979  assert(!nodesmask.get(node1));
980  assert(!nodesmask.get(partition1[i]));
981  nodesmask.set(node1);
982  nodesmask.set(partition1[i]);
983  }
984 
985  ndbout_c("done");
986 
987  if (restarter.restartAll(false, true, false))
988  return NDBT_FAILED;
989 
990  int dump[255];
991  dump[0] = 9000;
992  memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
993  for (Uint32 i = 0; i<cnt/2; i++)
994  if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2))
995  return NDBT_FAILED;
996 
997  dump[0] = 9000;
998  memcpy(dump + 1, partition1, sizeof(int)*cnt/2);
999  for (Uint32 i = 0; i<cnt/2; i++)
1000  if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2))
1001  return NDBT_FAILED;
1002 
1003  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1004 
1005  if (restarter.dumpStateAllNodes(val2, 2))
1006  return NDBT_FAILED;
1007 
1008  if (restarter.insertErrorInAllNodes(932))
1009  return NDBT_FAILED;
1010 
1011  if (restarter.startAll())
1012  return NDBT_FAILED;
1013 
1014  if (restarter.waitClusterStartPhase(2))
1015  return NDBT_FAILED;
1016 
1017  dump[0] = 9001;
1018  for (Uint32 i = 0; i<cnt/2; i++)
1019  if (restarter.dumpStateAllNodes(dump, 2))
1020  return NDBT_FAILED;
1021 
1022  if (restarter.waitClusterNoStart(30))
1023  if (restarter.waitNodesNoStart(partition0, cnt/2, 10))
1024  if (restarter.waitNodesNoStart(partition1, cnt/2, 10))
1025  return NDBT_FAILED;
1026 
1027  if (restarter.startAll())
1028  return NDBT_FAILED;
1029 
1030  if (restarter.waitClusterStarted())
1031  return NDBT_FAILED;
1032  }
1033  return NDBT_OK;
1034 }
1035 
1036 int runBug20185(NDBT_Context* ctx, NDBT_Step* step)
1037 {
1038  NdbRestarter restarter;
1039  HugoOperations hugoOps(*ctx->getTab());
1040  Ndb* pNdb = GETNDB(step);
1041 
1042  const int masterNode = restarter.getMasterNodeId();
1043 
1044  int dump[] = { 7090, 20 } ;
1045  if (restarter.dumpStateAllNodes(dump, 2))
1046  return NDBT_FAILED;
1047 
1048  NdbSleep_MilliSleep(3000);
1049  Vector<int> nodes;
1050  for (int i = 0; i<restarter.getNumDbNodes(); i++)
1051  nodes.push_back(restarter.getDbNodeId(i));
1052 
1053 retry:
1054  if(hugoOps.startTransaction(pNdb) != 0)
1055  return NDBT_FAILED;
1056 
1057  if(hugoOps.pkUpdateRecord(pNdb, 1, 1) != 0)
1058  return NDBT_FAILED;
1059 
1060  if (hugoOps.execute_NoCommit(pNdb) != 0)
1061  return NDBT_FAILED;
1062 
1063  const int node = hugoOps.getTransaction()->getConnectedNodeId();
1064  if (node != masterNode)
1065  {
1066  hugoOps.closeTransaction(pNdb);
1067  goto retry;
1068  }
1069 
1070  int nodeId;
1071  do {
1072  nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
1073  } while (nodeId == node);
1074 
1075  ndbout_c("7031 to %d", nodeId);
1076  if (restarter.insertErrorInNode(nodeId, 7031))
1077  return NDBT_FAILED;
1078 
1079  for (Uint32 i = 0; i<nodes.size(); i++)
1080  {
1081  if (nodes[i] != nodeId)
1082  if (restarter.insertErrorInNode(nodes[i], 7030))
1083  return NDBT_FAILED;
1084  }
1085 
1086  NdbSleep_MilliSleep(500);
1087 
1088  if (hugoOps.execute_Commit(pNdb) == 0)
1089  return NDBT_FAILED;
1090 
1091  NdbSleep_MilliSleep(3000);
1092 
1093  restarter.waitClusterStarted();
1094 
1095  if (restarter.dumpStateAllNodes(dump, 1))
1096  return NDBT_FAILED;
1097 
1098  return NDBT_OK;
1099 }
1100 
1101 int runBug24717(NDBT_Context* ctx, NDBT_Step* step)
1102 {
1103  int loops = ctx->getNumLoops();
1104  NdbRestarter restarter;
1105  Ndb* pNdb = GETNDB(step);
1106 
1107  HugoTransactions hugoTrans(*ctx->getTab());
1108 
1109  int dump[] = { 9002, 0 } ;
1110  Uint32 ownNode = refToNode(pNdb->getReference());
1111  dump[1] = ownNode;
1112 
1113  for (; loops; loops --)
1114  {
1115  int nodeId = restarter.getRandomNotMasterNodeId(rand());
1116  restarter.restartOneDbNode(nodeId, false, true, true);
1117  restarter.waitNodesNoStart(&nodeId, 1);
1118 
1119  if (restarter.dumpStateOneNode(nodeId, dump, 2))
1120  return NDBT_FAILED;
1121 
1122  restarter.startNodes(&nodeId, 1);
1123 
1124  do {
1125  for (Uint32 i = 0; i < 100; i++)
1126  {
1127  hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead);
1128  }
1129  } while (restarter.waitClusterStarted(5) != 0);
1130  }
1131 
1132  return NDBT_OK;
1133 }
1134 
1135 int
1136 runBug29364(NDBT_Context* ctx, NDBT_Step* step)
1137 {
1138  int loops = ctx->getNumLoops();
1139  NdbRestarter restarter;
1140  Ndb* pNdb = GETNDB(step);
1141 
1142  HugoTransactions hugoTrans(*ctx->getTab());
1143 
1144  if (restarter.getNumDbNodes() < 4)
1145  return NDBT_OK;
1146 
1147  int dump0[] = { 9000, 0 } ;
1148  int dump1[] = { 9001, 0 } ;
1149  Uint32 ownNode = refToNode(pNdb->getReference());
1150  dump0[1] = ownNode;
1151 
1152  for (; loops; loops --)
1153  {
1154  int node0 = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
1155  int node1 = restarter.getRandomNodeOtherNodeGroup(node0, rand());
1156 
1157  restarter.restartOneDbNode(node0, false, true, true);
1158  restarter.waitNodesNoStart(&node0, 1);
1159  restarter.startNodes(&node0, 1);
1160  restarter.waitClusterStarted();
1161 
1162  restarter.restartOneDbNode(node1, false, true, true);
1163  restarter.waitNodesNoStart(&node1, 1);
1164  if (restarter.dumpStateOneNode(node1, dump0, 2))
1165  return NDBT_FAILED;
1166 
1167  restarter.startNodes(&node1, 1);
1168 
1169  do {
1170 
1171  for (Uint32 i = 0; i < 100; i++)
1172  {
1173  hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead);
1174  }
1175  } while (restarter.waitClusterStarted(5) != 0);
1176 
1177  if (restarter.dumpStateOneNode(node1, dump1, 1))
1178  return NDBT_FAILED;
1179  }
1180 
1181  return NDBT_OK;
1182 }
1183 
1184 int runBug25364(NDBT_Context* ctx, NDBT_Step* step)
1185 {
1186  NdbRestarter restarter;
1187  int loops = ctx->getNumLoops();
1188 
1189  if (restarter.getNumDbNodes() < 4)
1190  return NDBT_OK;
1191 
1192  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1193 
1194  for (; loops; loops --)
1195  {
1196  int master = restarter.getMasterNodeId();
1197  int victim = restarter.getRandomNodeOtherNodeGroup(master, rand());
1198  int second = restarter.getRandomNodeSameNodeGroup(victim, rand());
1199 
1200  int dump[] = { 935, victim } ;
1201  if (restarter.dumpStateOneNode(master, dump, 2))
1202  return NDBT_FAILED;
1203 
1204  if (restarter.dumpStateOneNode(master, val2, 2))
1205  return NDBT_FAILED;
1206 
1207  if (restarter.restartOneDbNode(second, false, true, true))
1208  return NDBT_FAILED;
1209 
1210  int nodes[2] = { master, second };
1211  if (restarter.waitNodesNoStart(nodes, 2))
1212  return NDBT_FAILED;
1213 
1214  restarter.startNodes(nodes, 2);
1215 
1216  if (restarter.waitNodesStarted(nodes, 2))
1217  return NDBT_FAILED;
1218  }
1219 
1220  return NDBT_OK;
1221 }
1222 
1223 int
1224 runBug21271(NDBT_Context* ctx, NDBT_Step* step)
1225 {
1226  NdbRestarter restarter;
1227  HugoOperations hugoOps(*ctx->getTab());
1228 
1229  const int masterNode = restarter.getMasterNodeId();
1230  const int nodeId = restarter.getRandomNodeSameNodeGroup(masterNode, rand());
1231 
1232  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1233  if (restarter.dumpStateOneNode(nodeId, val2, 2))
1234  return NDBT_FAILED;
1235 
1236  Uint32 tableId = ctx->getTab()->getTableId();
1237  int dump[] = { DumpStateOrd::LqhErrorInsert5042, 0, 5044 };
1238  dump[1] = tableId;
1239 
1240  if (restarter.dumpStateOneNode(nodeId, dump, 3))
1241  return NDBT_FAILED;
1242 
1243  restarter.waitNodesNoStart(&nodeId, 1);
1244  ctx->stopTest();
1245 
1246  restarter.startNodes(&nodeId, 1);
1247 
1248  if (restarter.waitClusterStarted() != 0)
1249  return NDBT_FAILED;
1250 
1251  return NDBT_OK;
1252  return NDBT_OK;
1253 }
1254 
1255 int
1256 runBug24543(NDBT_Context* ctx, NDBT_Step* step){
1257  NdbRestarter restarter;
1258 
1259  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1260  if (restarter.dumpStateAllNodes(val2, 2))
1261  return NDBT_FAILED;
1262 
1263  int nodes[2];
1264  nodes[0] = restarter.getMasterNodeId();
1265  restarter.insertErrorInNode(nodes[0], 934);
1266 
1267  nodes[1] = restarter.getRandomNodeOtherNodeGroup(nodes[0], rand());
1268  if (nodes[1] == -1)
1269  {
1270  nodes[1] = restarter.getRandomNodeSameNodeGroup(nodes[0], rand());
1271  }
1272 
1273  restarter.restartOneDbNode(nodes[1], false, true, true);
1274  if (restarter.waitNodesNoStart(nodes, 2))
1275  return NDBT_FAILED;
1276 
1277  restarter.startNodes(nodes, 2);
1278  if (restarter.waitNodesStarted(nodes, 2))
1279  {
1280  return NDBT_FAILED;
1281  }
1282  return NDBT_OK;
1283 }
1284 
1285 int runBug25468(NDBT_Context* ctx, NDBT_Step* step)
1286 {
1287  int loops = ctx->getNumLoops();
1288  NdbRestarter restarter;
1289 
1290  for (int i = 0; i<loops; i++)
1291  {
1292  int master = restarter.getMasterNodeId();
1293  int node1, node2;
1294  switch(i % 5){
1295  case 0:
1296  node1 = master;
1297  node2 = restarter.getRandomNodeSameNodeGroup(master, rand());
1298  break;
1299  case 1:
1300  node1 = restarter.getRandomNodeSameNodeGroup(master, rand());
1301  node2 = master;
1302  break;
1303  case 2:
1304  case 3:
1305  case 4:
1306  node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
1307  if (node1 == -1)
1308  node1 = master;
1309  node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
1310  break;
1311  }
1312 
1313  ndbout_c("node1: %d node2: %d master: %d", node1, node2, master);
1314 
1315  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1316 
1317  if (restarter.dumpStateOneNode(node2, val2, 2))
1318  return NDBT_FAILED;
1319 
1320  if (restarter.insertErrorInNode(node1, 7178))
1321  return NDBT_FAILED;
1322 
1323  int val1 = 7099;
1324  if (restarter.dumpStateOneNode(master, &val1, 1))
1325  return NDBT_FAILED;
1326 
1327  if (restarter.waitNodesNoStart(&node2, 1))
1328  return NDBT_FAILED;
1329 
1330  if (restarter.startAll())
1331  return NDBT_FAILED;
1332 
1333  if (restarter.waitClusterStarted())
1334  return NDBT_FAILED;
1335  }
1336 
1337  return NDBT_OK;
1338 }
1339 
1340 int runBug25554(NDBT_Context* ctx, NDBT_Step* step)
1341 {
1342  int loops = ctx->getNumLoops();
1343  NdbRestarter restarter;
1344 
1345  if (restarter.getNumDbNodes() < 4)
1346  return NDBT_OK;
1347 
1348  for (int i = 0; i<loops; i++)
1349  {
1350  int master = restarter.getMasterNodeId();
1351  int node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
1352  restarter.restartOneDbNode(node1, false, true, true);
1353 
1354  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1355 
1356  if (restarter.dumpStateOneNode(master, val2, 2))
1357  return NDBT_FAILED;
1358 
1359  if (restarter.insertErrorInNode(master, 7141))
1360  return NDBT_FAILED;
1361 
1362  if (restarter.waitNodesNoStart(&node1, 1))
1363  return NDBT_FAILED;
1364 
1365  if (restarter.dumpStateOneNode(node1, val2, 2))
1366  return NDBT_FAILED;
1367 
1368  if (restarter.insertErrorInNode(node1, 932))
1369  return NDBT_FAILED;
1370 
1371  if (restarter.startNodes(&node1, 1))
1372  return NDBT_FAILED;
1373 
1374  int nodes[] = { master, node1 };
1375  if (restarter.waitNodesNoStart(nodes, 2))
1376  return NDBT_FAILED;
1377 
1378  if (restarter.startNodes(nodes, 2))
1379  return NDBT_FAILED;
1380 
1381  if (restarter.waitClusterStarted())
1382  return NDBT_FAILED;
1383  }
1384 
1385  return NDBT_OK;
1386 }
1387 
1388 int runBug25984(NDBT_Context* ctx, NDBT_Step* step)
1389 {
1390  NdbRestarter restarter;
1391  Ndb* pNdb = GETNDB(step);
1392 
1393  NdbDictionary::Table tab = * ctx->getTab();
1394  NdbDictionary::Dictionary* pDict = GETNDB(step)->getDictionary();
1395 
1396  if (restarter.getNumDbNodes() < 2)
1397  return NDBT_OK;
1398 
1399  pDict->dropTable(tab.getName());
1400 
1401  if (restarter.restartAll(true, true, true))
1402  return NDBT_FAILED;
1403 
1404  if (restarter.waitClusterNoStart())
1405  return NDBT_FAILED;
1406 
1407  if (restarter.startAll())
1408  return NDBT_FAILED;
1409 
1410  if (restarter.waitClusterStarted())
1411  return NDBT_FAILED;
1412 
1413  int res = pDict->createTable(tab);
1414  if (res)
1415  {
1416  return NDBT_FAILED;
1417  }
1418  HugoTransactions trans(* pDict->getTable(tab.getName()));
1419  trans.loadTable(pNdb, ctx->getNumRecords());
1420 
1421  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1422  int master = restarter.getMasterNodeId();
1423  int victim = restarter.getRandomNodeOtherNodeGroup(master, rand());
1424  if (victim == -1)
1425  victim = restarter.getRandomNodeSameNodeGroup(master, rand());
1426 
1427  restarter.restartOneDbNode(victim, false, true, true);
1428 
1429  for (Uint32 i = 0; i<10; i++)
1430  {
1431  ndbout_c("Loop: %d", i);
1432  if (restarter.waitNodesNoStart(&victim, 1))
1433  return NDBT_FAILED;
1434 
1435  if (restarter.dumpStateOneNode(victim, val2, 2))
1436  return NDBT_FAILED;
1437 
1438  if (restarter.insertErrorInNode(victim, 7191))
1439  return NDBT_FAILED;
1440 
1441  trans.scanUpdateRecords(pNdb, ctx->getNumRecords());
1442 
1443  if (restarter.startNodes(&victim, 1))
1444  return NDBT_FAILED;
1445 
1446  NdbSleep_SecSleep(3);
1447  }
1448 
1449  if (restarter.waitNodesNoStart(&victim, 1))
1450  return NDBT_FAILED;
1451 
1452  if (restarter.restartAll(false, false, true))
1453  return NDBT_FAILED;
1454 
1455  if (restarter.waitClusterStarted())
1456  return NDBT_FAILED;
1457 
1458  trans.scanUpdateRecords(pNdb, ctx->getNumRecords());
1459 
1460  restarter.restartOneDbNode(victim, false, true, true);
1461  for (Uint32 i = 0; i<1; i++)
1462  {
1463  ndbout_c("Loop: %d", i);
1464  if (restarter.waitNodesNoStart(&victim, 1))
1465  return NDBT_FAILED;
1466 
1467  if (restarter.dumpStateOneNode(victim, val2, 2))
1468  return NDBT_FAILED;
1469 
1470  if (restarter.insertErrorInNode(victim, 7016))
1471  return NDBT_FAILED;
1472 
1473  trans.scanUpdateRecords(pNdb, ctx->getNumRecords());
1474 
1475  if (restarter.startNodes(&victim, 1))
1476  return NDBT_FAILED;
1477 
1478  NdbSleep_SecSleep(3);
1479  }
1480 
1481  if (restarter.waitNodesNoStart(&victim, 1))
1482  return NDBT_FAILED;
1483 
1484  if (restarter.startNodes(&victim, 1))
1485  return NDBT_FAILED;
1486 
1487  if (restarter.waitClusterStarted())
1488  return NDBT_FAILED;
1489 
1490  return NDBT_OK;
1491 }
1492 
1493 int
1494 runBug26457(NDBT_Context* ctx, NDBT_Step* step)
1495 {
1496  NdbRestarter res;
1497  if (res.getNumDbNodes() < 4)
1498  return NDBT_OK;
1499 
1500  int loops = ctx->getNumLoops();
1501  while (loops --)
1502  {
1503 retry:
1504  int master = res.getMasterNodeId();
1505  int next = res.getNextMasterNodeId(master);
1506 
1507  ndbout_c("master: %d next: %d", master, next);
1508 
1509  if (res.getNodeGroup(master) == res.getNodeGroup(next))
1510  {
1511  res.restartOneDbNode(next, false, false, true);
1512  if (res.waitClusterStarted())
1513  return NDBT_FAILED;
1514  goto retry;
1515  }
1516 
1517  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 2 };
1518 
1519  if (res.dumpStateOneNode(next, val2, 2))
1520  return NDBT_FAILED;
1521 
1522  if (res.insertErrorInNode(next, 7180))
1523  return NDBT_FAILED;
1524 
1525  res.restartOneDbNode(master, false, false, true);
1526  if (res.waitClusterStarted())
1527  return NDBT_FAILED;
1528  }
1529 
1530  return NDBT_OK;
1531 }
1532 
1533 int
1534 runBug26481(NDBT_Context* ctx, NDBT_Step* step)
1535 {
1536  NdbRestarter res;
1537 
1538  int node = res.getRandomNotMasterNodeId(rand());
1539  ndbout_c("node: %d", node);
1540  if (res.restartOneDbNode(node, true, true, true))
1541  return NDBT_FAILED;
1542 
1543  if (res.waitNodesNoStart(&node, 1))
1544  return NDBT_FAILED;
1545 
1546  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1547  if (res.dumpStateOneNode(node, val2, 2))
1548  return NDBT_FAILED;
1549 
1550  if (res.insertErrorInNode(node, 7018))
1551  return NDBT_FAILED;
1552 
1553  if (res.startNodes(&node, 1))
1554  return NDBT_FAILED;
1555 
1556  res.waitNodesStartPhase(&node, 1, 3);
1557 
1558  if (res.waitNodesNoStart(&node, 1))
1559  return NDBT_FAILED;
1560 
1561  res.startNodes(&node, 1);
1562 
1563  if (res.waitClusterStarted())
1564  return NDBT_FAILED;
1565 
1566  return NDBT_OK;
1567 }
1568 
1569 int
1570 runBug26450(NDBT_Context* ctx, NDBT_Step* step)
1571 {
1572  NdbRestarter res;
1573  Ndb* pNdb = GETNDB(step);
1574 
1575  int node = res.getRandomNotMasterNodeId(rand());
1576  Vector<int> nodes;
1577  for (int i = 0; i<res.getNumDbNodes(); i++)
1578  {
1579  if (res.getDbNodeId(i) != node)
1580  nodes.push_back(res.getDbNodeId(i));
1581  }
1582 
1583  if (res.restartAll())
1584  return NDBT_FAILED;
1585 
1586  if (res.waitClusterStarted())
1587  return NDBT_FAILED;
1588 
1589  ndbout_c("node: %d", node);
1590  if (res.restartOneDbNode(node, false, true, true))
1591  return NDBT_FAILED;
1592 
1593  if (res.waitNodesNoStart(&node, 1))
1594  return NDBT_FAILED;
1595 
1596  if (runClearTable(ctx, step))
1597  return NDBT_FAILED;
1598 
1599  for (int i = 0; i < 2; i++)
1600  {
1601  if (res.restartAll(false, true, i > 0))
1602  return NDBT_FAILED;
1603 
1604  if (res.waitClusterNoStart())
1605  return NDBT_FAILED;
1606 
1607  if (res.startNodes(nodes.getBase(), nodes.size()))
1608  return NDBT_FAILED;
1609 
1610  if (res.waitNodesStarted(nodes.getBase(), nodes.size()))
1611  return NDBT_FAILED;
1612  }
1613 
1614  if (res.startNodes(&node, 1))
1615  return NDBT_FAILED;
1616 
1617  if (res.waitNodesStarted(&node, 1))
1618  return NDBT_FAILED;
1619 
1620  HugoTransactions trans (* ctx->getTab());
1621  if (trans.selectCount(pNdb) != 0)
1622  return NDBT_FAILED;
1623 
1624  return NDBT_OK;
1625 }
1626 
1627 int
1628 runBug27003(NDBT_Context* ctx, NDBT_Step* step)
1629 {
1630  int loops = ctx->getNumLoops();
1631  NdbRestarter res;
1632 
1633  static const int errnos[] = { 4025, 4026, 4027, 4028, 0 };
1634 
1635  int node = res.getRandomNotMasterNodeId(rand());
1636  ndbout_c("node: %d", node);
1637  if (res.restartOneDbNode(node, true, true, true))
1638  return NDBT_FAILED;
1639 
1640  Uint32 pos = 0;
1641  for (int i = 0; i<loops; i++)
1642  {
1643  while (errnos[pos] != 0)
1644  {
1645  ndbout_c("Testing err: %d", errnos[pos]);
1646 
1647  if (res.waitNodesNoStart(&node, 1))
1648  return NDBT_FAILED;
1649 
1650  if (res.insertErrorInNode(node, 1000))
1651  return NDBT_FAILED;
1652 
1653  if (res.insertErrorInNode(node, errnos[pos]))
1654  return NDBT_FAILED;
1655 
1656  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 3 };
1657  if (res.dumpStateOneNode(node, val2, 2))
1658  return NDBT_FAILED;
1659 
1660  res.startNodes(&node, 1);
1661  NdbSleep_SecSleep(3);
1662  pos++;
1663  }
1664  pos = 0;
1665  }
1666 
1667  if (res.waitNodesNoStart(&node, 1))
1668  return NDBT_FAILED;
1669 
1670  res.startNodes(&node, 1);
1671  if (res.waitClusterStarted())
1672  return NDBT_FAILED;
1673 
1674  return NDBT_OK;
1675 }
1676 
1677 
1678 int
1679 runBug27283(NDBT_Context* ctx, NDBT_Step* step)
1680 {
1681  int loops = ctx->getNumLoops();
1682  NdbRestarter res;
1683 
1684  if (res.getNumDbNodes() < 2)
1685  {
1686  return NDBT_OK;
1687  }
1688 
1689  static const int errnos[] = { 7181, 7182, 0 };
1690 
1691  Uint32 pos = 0;
1692  for (Uint32 i = 0; i<(Uint32)loops; i++)
1693  {
1694  while (errnos[pos] != 0)
1695  {
1696  int master = res.getMasterNodeId();
1697  int next = res.getNextMasterNodeId(master);
1698  //int next2 = res.getNextMasterNodeId(next);
1699 
1700  //int node = (i & 1) ? next : next2;
1701  ndbout_c("Testing err: %d", errnos[pos]);
1702  if (res.insertErrorInNode(next, errnos[pos]))
1703  return NDBT_FAILED;
1704 
1705  NdbSleep_SecSleep(3);
1706 
1707  if (res.waitClusterStarted())
1708  return NDBT_FAILED;
1709 
1710  pos++;
1711  }
1712  pos = 0;
1713  }
1714 
1715  return NDBT_OK;
1716 }
1717 
1718 int
1719 runBug27466(NDBT_Context* ctx, NDBT_Step* step)
1720 {
1721  int loops = ctx->getNumLoops();
1722  NdbRestarter res;
1723 
1724  if (res.getNumDbNodes() < 2)
1725  {
1726  return NDBT_OK;
1727  }
1728 
1729  for (Uint32 i = 0; i<(Uint32)loops; i++)
1730  {
1731  int node1 = res.getDbNodeId(rand() % res.getNumDbNodes());
1732  int node2 = node1;
1733  while (node1 == node2)
1734  {
1735  node2 = res.getDbNodeId(rand() % res.getNumDbNodes());
1736  }
1737 
1738  ndbout_c("nodes %u %u", node1, node2);
1739 
1740  if (res.restartOneDbNode(node1, false, true, true))
1741  return NDBT_FAILED;
1742 
1743  if (res.waitNodesNoStart(&node1, 1))
1744  return NDBT_FAILED;
1745 
1746  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1747  if (res.dumpStateOneNode(node1, val2, 2))
1748  return NDBT_FAILED;
1749 
1750  if (res.insertErrorInNode(node2, 8039))
1751  return NDBT_FAILED;
1752 
1753  res.startNodes(&node1, 1);
1754  NdbSleep_SecSleep(3);
1755  if (res.waitNodesNoStart(&node1, 1))
1756  return NDBT_FAILED;
1757  NdbSleep_SecSleep(5); // Wait for delayed INCL_NODECONF to arrive
1758 
1759  res.startNodes(&node1, 1);
1760  if (res.waitClusterStarted())
1761  return NDBT_FAILED;
1762  }
1763 
1764  return NDBT_OK;
1765 }
1766 
1767 int
1768 runBug28023(NDBT_Context* ctx, NDBT_Step* step)
1769 {
1770  int loops = ctx->getNumLoops();
1771  int records = ctx->getNumRecords();
1772  Ndb* pNdb = GETNDB(step);
1773  NdbRestarter res;
1774 
1775  if (res.getNumDbNodes() < 2)
1776  {
1777  return NDBT_OK;
1778  }
1779 
1780 
1781  HugoTransactions hugoTrans(*ctx->getTab());
1782  if (hugoTrans.loadTable(pNdb, records) != 0){
1783  return NDBT_FAILED;
1784  }
1785 
1786  if (hugoTrans.clearTable(pNdb, records) != 0)
1787  {
1788  return NDBT_FAILED;
1789  }
1790 
1791  for (Uint32 i = 0; i<(Uint32)loops; i++)
1792  {
1793  int node1 = res.getDbNodeId(rand() % res.getNumDbNodes());
1794 
1795  if (res.restartOneDbNode2(node1,
1796  NdbRestarter::NRRF_ABORT |
1797  NdbRestarter::NRRF_NOSTART))
1798  return NDBT_FAILED;
1799 
1800  if (res.waitNodesNoStart(&node1, 1))
1801  return NDBT_FAILED;
1802 
1803  if (hugoTrans.loadTable(pNdb, records) != 0){
1804  return NDBT_FAILED;
1805  }
1806 
1807  if (hugoTrans.clearTable(pNdb, records) != 0)
1808  {
1809  return NDBT_FAILED;
1810  }
1811 
1812  res.startNodes(&node1, 1);
1813  if (res.waitClusterStarted())
1814  return NDBT_FAILED;
1815 
1816  if (hugoTrans.loadTable(pNdb, records) != 0){
1817  return NDBT_FAILED;
1818  }
1819 
1820  if (hugoTrans.scanUpdateRecords(pNdb, records) != 0)
1821  return NDBT_FAILED;
1822 
1823  if (hugoTrans.clearTable(pNdb, records) != 0)
1824  {
1825  return NDBT_FAILED;
1826  }
1827  }
1828 
1829  return NDBT_OK;
1830 }
1831 
1832 
1833 int
1834 runBug28717(NDBT_Context* ctx, NDBT_Step* step)
1835 {
1836  NdbRestarter res;
1837 
1838  if (res.getNumDbNodes() < 4)
1839  {
1840  return NDBT_OK;
1841  }
1842 
1843  int master = res.getMasterNodeId();
1844  int node0 = res.getRandomNodeOtherNodeGroup(master, rand());
1845  int node1 = res.getRandomNodeSameNodeGroup(node0, rand());
1846 
1847  ndbout_c("master: %d node0: %d node1: %d", master, node0, node1);
1848 
1849  if (res.restartOneDbNode(node0, false, true, true))
1850  {
1851  return NDBT_FAILED;
1852  }
1853 
1854  {
1855  int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
1856  NdbLogEventHandle handle =
1857  ndb_mgm_create_logevent_handle(res.handle, filter);
1858 
1859 
1860  int dump[] = { DumpStateOrd::DihStartLcpImmediately };
1861  struct ndb_logevent event;
1862 
1863  for (Uint32 i = 0; i<3; i++)
1864  {
1865  res.dumpStateOneNode(master, dump, 1);
1866  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1867  event.type != NDB_LE_LocalCheckpointStarted);
1868  while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1869  event.type != NDB_LE_LocalCheckpointCompleted);
1870  }
1871  }
1872 
1873  if (res.waitNodesNoStart(&node0, 1))
1874  return NDBT_FAILED;
1875 
1876  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1877 
1878  if (res.dumpStateOneNode(node0, val2, 2))
1879  return NDBT_FAILED;
1880 
1881  if (res.insertErrorInNode(node0, 5010))
1882  return NDBT_FAILED;
1883 
1884  if (res.insertErrorInNode(node1, 1001))
1885  return NDBT_FAILED;
1886 
1887  if (res.startNodes(&node0, 1))
1888  return NDBT_FAILED;
1889 
1890  NdbSleep_SecSleep(3);
1891 
1892  if (res.insertErrorInNode(node1, 0))
1893  return NDBT_FAILED;
1894 
1895  if (res.waitNodesNoStart(&node0, 1))
1896  return NDBT_FAILED;
1897 
1898  if (res.startNodes(&node0, 1))
1899  return NDBT_FAILED;
1900 
1901  if (res.waitClusterStarted())
1902  return NDBT_FAILED;
1903 
1904  return NDBT_OK;
1905 }
1906 
1907 static
1908 int
1909 f_master_failure [] = {
1910  7000, 7001, 7002, 7003, 7004, 7186, 7187, 7188, 7189, 7190, 0
1911 };
1912 
1913 static
1914 int
1915 f_participant_failure [] = {
1916  7005, 7006, 7007, 7008, 5000, 7228, 0
1917 };
1918 
1919 int
1920 runerrors(NdbRestarter& res, NdbRestarter::NodeSelector sel, const int* errors)
1921 {
1922  for (Uint32 i = 0; errors[i]; i++)
1923  {
1924  int node = res.getNode(sel);
1925 
1926  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1927  if (res.dumpStateOneNode(node, val2, 2))
1928  return NDBT_FAILED;
1929 
1930  ndbout << "node " << node << " err: " << errors[i]<< endl;
1931  if (res.insertErrorInNode(node, errors[i]))
1932  return NDBT_FAILED;
1933 
1934  if (res.waitNodesNoStart(&node, 1) != 0)
1935  return NDBT_FAILED;
1936 
1937  res.startNodes(&node, 1);
1938 
1939  if (res.waitClusterStarted() != 0)
1940  return NDBT_FAILED;
1941  }
1942  return NDBT_OK;
1943 }
1944 
1945 int
1946 runGCP(NDBT_Context* ctx, NDBT_Step* step)
1947 {
1948  NdbRestarter res;
1949  int loops = ctx->getNumLoops();
1950 
1951  if (res.getNumDbNodes() < 2)
1952  {
1953  return NDBT_OK;
1954  }
1955 
1956  if (res.getNumDbNodes() < 4)
1957  {
1961  Uint32 i;
1962  for (i = 0; f_master_failure[i] && f_master_failure[i] != 7186; i++);
1963  f_master_failure[i] = 0;
1964  }
1965 
1966  while (loops >= 0 && !ctx->isTestStopped())
1967  {
1968  loops --;
1969 
1970 #if 0
1971  if (runerrors(res, NdbRestarter::NS_NON_MASTER, f_participant_failure))
1972  {
1973  return NDBT_FAILED;
1974  }
1975 
1976  if (runerrors(res, NdbRestarter::NS_MASTER, f_participant_failure))
1977  {
1978  return NDBT_FAILED;
1979  }
1980 #endif
1981 
1982  if (runerrors(res, NdbRestarter::NS_RANDOM, f_participant_failure))
1983  {
1984  return NDBT_FAILED;
1985  }
1986 
1987  if (runerrors(res, NdbRestarter::NS_MASTER, f_master_failure))
1988  {
1989  return NDBT_FAILED;
1990  }
1991  }
1992  ctx->stopTest();
1993  return NDBT_OK;
1994 }
1995 
1996 int
1997 runCommitAck(NDBT_Context* ctx, NDBT_Step* step)
1998 {
1999  int loops = ctx->getNumLoops();
2000  int records = ctx->getNumRecords();
2001  NdbRestarter restarter;
2002  Ndb* pNdb = GETNDB(step);
2003 
2004  if (records < 2)
2005  return NDBT_OK;
2006  if (restarter.getNumDbNodes() < 2)
2007  return NDBT_OK;
2008 
2009  int trans_type= -1;
2010  NdbConnection *pCon;
2011  int node;
2012  while (loops--)
2013  {
2014  trans_type++;
2015  if (trans_type > 2)
2016  trans_type= 0;
2017  HugoTransactions hugoTrans(*ctx->getTab());
2018  switch (trans_type) {
2019  case 0:
2020  /*
2021  - load records less 1
2022  */
2023  g_info << "case 0\n";
2024  if (hugoTrans.loadTable(GETNDB(step), records - 1))
2025  {
2026  return NDBT_FAILED;
2027  }
2028  break;
2029  case 1:
2030  /*
2031  - load 1 record
2032  */
2033  g_info << "case 1\n";
2034  if (hugoTrans.loadTable(GETNDB(step), 1))
2035  {
2036  return NDBT_FAILED;
2037  }
2038  break;
2039  case 2:
2040  /*
2041  - load 1 record in the end
2042  */
2043  g_info << "case 2\n";
2044  {
2045  HugoOperations hugoOps(*ctx->getTab());
2046  if (hugoOps.startTransaction(pNdb))
2047  abort();
2048  if (hugoOps.pkInsertRecord(pNdb, records-1))
2049  abort();
2050  if (hugoOps.execute_Commit(pNdb))
2051  abort();
2052  if (hugoOps.closeTransaction(pNdb))
2053  abort();
2054  }
2055  break;
2056  default:
2057  abort();
2058  }
2059 
2060  /* run transaction that should be tested */
2061  HugoOperations hugoOps(*ctx->getTab());
2062  if (hugoOps.startTransaction(pNdb))
2063  return NDBT_FAILED;
2064  pCon= hugoOps.getTransaction();
2065  node= pCon->getConnectedNodeId();
2066  switch (trans_type) {
2067  case 0:
2068  case 1:
2069  /*
2070  insert records with ignore error
2071  - insert rows, some exist already
2072  */
2073  for (int i= 0; i < records; i++)
2074  {
2075  if (hugoOps.pkInsertRecord(pNdb, i))
2076  goto err;
2077  }
2078  break;
2079  case 2:
2080  /*
2081  insert records with ignore error
2082  - insert rows, some exist already
2083  */
2084  for (int i= 0; i < records; i++)
2085  {
2086  if (hugoOps.pkInsertRecord(pNdb, i))
2087  goto err;
2088  }
2089  break;
2090  default:
2091  abort();
2092  }
2093 
2094  /*
2095  insert error in ndb kernel (TC) that throws away acknowledge of commit
2096  and then die 5 seconds later
2097  */
2098  {
2099  if (restarter.insertErrorInNode(node, 8054))
2100  goto err;
2101  }
2102  {
2103  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2104  if (restarter.dumpStateOneNode(node, val2, 2))
2105  goto err;
2106  }
2107 
2108  /* execute transaction and verify return code */
2109  g_info << " execute... hangs for 5 seconds\n";
2110  {
2111  const NdbOperation *first= pCon->getFirstDefinedOperation();
2112  int check= pCon->execute(Commit, AO_IgnoreError);
2113  const NdbError err = pCon->getNdbError();
2114 
2115  while (first)
2116  {
2117  const NdbError &err= first->getNdbError();
2118  g_info << " error " << err.code << endl;
2119  first= pCon->getNextCompletedOperation(first);
2120  }
2121 
2122  int expected_commit_res[3]= { 630, 630, 630 };
2123  if (check == -1 ||
2124  err.code != expected_commit_res[trans_type])
2125  {
2126  g_err << "check == " << check << endl;
2127  g_err << "got error: "
2128  << err.code
2129  << " expected: "
2130  << expected_commit_res[trans_type]
2131  << endl;
2132  goto err;
2133  }
2134  }
2135 
2136  g_info << " wait node nostart\n";
2137  if (restarter.waitNodesNoStart(&node, 1))
2138  {
2139  g_err << " wait node nostart failed\n";
2140  goto err;
2141  }
2142 
2143  /* close transaction */
2144  if (hugoOps.closeTransaction(pNdb))
2145  return NDBT_FAILED;
2146 
2147  /* commit ack marker pools should be empty */
2148  g_info << " dump pool status\n";
2149  {
2150  int dump[255];
2151  dump[0] = 2552;
2152  if (restarter.dumpStateAllNodes(dump, 1))
2153  return NDBT_FAILED;
2154  }
2155 
2156  /* wait for cluster to come up again */
2157  g_info << " wait cluster started\n";
2158  if (restarter.startNodes(&node, 1) ||
2159  restarter.waitNodesStarted(&node, 1))
2160  {
2161  g_err << "Cluster failed to start\n";
2162  return NDBT_FAILED;
2163  }
2164 
2165  /* verify data */
2166  g_info << " verifying\n";
2167  switch (trans_type) {
2168  case 0:
2169  case 1:
2170  case 2:
2171  /*
2172  insert records with ignore error
2173  - should have all records
2174  */
2175  if (hugoTrans.scanReadRecords(GETNDB(step), records, 0, 64) != 0){
2176  return NDBT_FAILED;
2177  }
2178  break;
2179  default:
2180  abort();
2181  }
2182 
2183  /* cleanup for next round in loop */
2184  g_info << " cleaning\n";
2185  if (hugoTrans.clearTable(GETNDB(step), records))
2186  {
2187  return NDBT_FAILED;
2188  }
2189  continue;
2190 err:
2191  hugoOps.closeTransaction(pNdb);
2192  return NDBT_FAILED;
2193  }
2194  return NDBT_OK;
2195 }
2196 
2197 int
2198 max_cnt(int arr[], int cnt)
2199 {
2200  int res = 0;
2201 
2202  for (int i = 0; i<cnt ; i++)
2203  {
2204  if (arr[i] > res)
2205  {
2206  res = arr[i];
2207  }
2208  }
2209  return res;
2210 }
2211 
2212 int
2213 runPnr(NDBT_Context* ctx, NDBT_Step* step)
2214 {
2215  int loops = ctx->getNumLoops();
2216  NdbRestarter res;
2217  bool lcp = ctx->getProperty("LCP", (unsigned)0);
2218 
2219  int nodegroups[MAX_NDB_NODES];
2220  bzero(nodegroups, sizeof(nodegroups));
2221 
2222  for (int i = 0; i<res.getNumDbNodes(); i++)
2223  {
2224  int node = res.getDbNodeId(i);
2225  nodegroups[res.getNodeGroup(node)]++;
2226  }
2227 
2228  for (int i = 0; i<MAX_NDB_NODES; i++)
2229  {
2230  if (nodegroups[i] && nodegroups[i] == 1)
2231  {
2235  ctx->stopTest();
2236  return NDBT_OK;
2237  }
2238  }
2239 
2240  for (int i = 0; i<loops && ctx->isTestStopped() == false; i++)
2241  {
2242  if (lcp)
2243  {
2244  int lcpdump = DumpStateOrd::DihMinTimeBetweenLCP;
2245  res.dumpStateAllNodes(&lcpdump, 1);
2246  }
2247 
2248  int ng_copy[MAX_NDB_NODES];
2249  memcpy(ng_copy, nodegroups, sizeof(ng_copy));
2250 
2251  Vector<int> nodes;
2252  printf("restarting ");
2253  while (max_cnt(ng_copy, MAX_NDB_NODES) > 1)
2254  {
2255  int node = res.getNode(NdbRestarter::NS_RANDOM);
2256  int ng = res.getNodeGroup(node);
2257  if (ng_copy[ng] > 1)
2258  {
2259  printf("%u ", node);
2260  nodes.push_back(node);
2261  ng_copy[ng]--;
2262  }
2263  }
2264  printf("\n");
2265 
2266  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2267  for (Uint32 j = 0; j<nodes.size(); j++)
2268  {
2269  res.dumpStateOneNode(nodes[j], val2, 2);
2270  }
2271 
2272  int kill[] = { 9999, 1000, 3000 };
2273  for (Uint32 j = 0; j<nodes.size(); j++)
2274  {
2275  res.dumpStateOneNode(nodes[j], kill, 3);
2276  }
2277 
2278  if (res.waitNodesNoStart(nodes.getBase(), nodes.size()))
2279  return NDBT_FAILED;
2280 
2281  if (res.startNodes(nodes.getBase(), nodes.size()))
2282  return NDBT_FAILED;
2283 
2284  if (res.waitClusterStarted())
2285  return NDBT_FAILED;
2286  }
2287 
2288  ctx->stopTest();
2289  return NDBT_OK;
2290 }
2291 
2292 int
2293 runCreateBigTable(NDBT_Context* ctx, NDBT_Step* step)
2294 {
2295  NdbDictionary::Table tab = *ctx->getTab();
2296  BaseString tmp;
2297  tmp.assfmt("_%s", tab.getName());
2298  tab.setName(tmp.c_str());
2299 
2300  NdbDictionary::Dictionary* pDict = GETNDB(step)->getDictionary();
2301  int res = pDict->createTable(tab);
2302  if (res)
2303  {
2304  return NDBT_FAILED;
2305  }
2306 
2307  const NdbDictionary::Table* pTab = pDict->getTable(tmp.c_str());
2308  if (pTab == 0)
2309  {
2310  return NDBT_FAILED;
2311  }
2312 
2313  int bytes = tab.getRowSizeInBytes();
2314  int size = 50*1024*1024; // 50Mb
2315  int rows = size / bytes;
2316 
2317  if (rows > 1000000)
2318  rows = 1000000;
2319 
2320  ndbout_c("Loading %u rows into %s", rows, tmp.c_str());
2321  Uint64 now = NdbTick_CurrentMillisecond();
2322  HugoTransactions hugoTrans(*pTab);
2323  int cnt = 0;
2324  do {
2325  hugoTrans.loadTableStartFrom(GETNDB(step), cnt, 10000);
2326  cnt += 10000;
2327  } while (cnt < rows && (NdbTick_CurrentMillisecond() - now) < 30000); //30s
2328  ndbout_c("Loaded %u rows in %llums", cnt,
2329  NdbTick_CurrentMillisecond() - now);
2330 
2331  return NDBT_OK;
2332 }
2333 
2334 int
2335 runDropBigTable(NDBT_Context* ctx, NDBT_Step* step)
2336 {
2337  NdbDictionary::Table tab = *ctx->getTab();
2338  BaseString tmp;
2339  tmp.assfmt("_%s", tab.getName());
2340  GETNDB(step)->getDictionary()->dropTable(tmp.c_str());
2341  return NDBT_OK;
2342 }
2343 
2344 int
2345 runBug31525(NDBT_Context* ctx, NDBT_Step* step)
2346 {
2347  //int result = NDBT_OK;
2348  //int loops = ctx->getNumLoops();
2349  //int records = ctx->getNumRecords();
2350  //Ndb* pNdb = GETNDB(step);
2351  NdbRestarter res;
2352 
2353  if (res.getNumDbNodes() < 2)
2354  {
2355  return NDBT_OK;
2356  }
2357 
2358  int nodes[2];
2359  nodes[0] = res.getMasterNodeId();
2360  nodes[1] = res.getNextMasterNodeId(nodes[0]);
2361 
2362  while (res.getNodeGroup(nodes[0]) != res.getNodeGroup(nodes[1]))
2363  {
2364  ndbout_c("Restarting %u as it not in same node group as %u",
2365  nodes[1], nodes[0]);
2366  if (res.restartOneDbNode(nodes[1], false, true, true))
2367  return NDBT_FAILED;
2368 
2369  if (res.waitNodesNoStart(nodes+1, 1))
2370  return NDBT_FAILED;
2371 
2372  if (res.startNodes(nodes+1, 1))
2373  return NDBT_FAILED;
2374 
2375  if (res.waitClusterStarted())
2376  return NDBT_FAILED;
2377 
2378  nodes[1] = res.getNextMasterNodeId(nodes[0]);
2379  }
2380 
2381  ndbout_c("nodes[0]: %u nodes[1]: %u", nodes[0], nodes[1]);
2382 
2383  int val = DumpStateOrd::DihMinTimeBetweenLCP;
2384  if (res.dumpStateAllNodes(&val, 1))
2385  return NDBT_FAILED;
2386 
2387  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2388  if (res.dumpStateAllNodes(val2, 2))
2389  return NDBT_FAILED;
2390 
2391  if (res.insertErrorInAllNodes(932))
2392  return NDBT_FAILED;
2393 
2394  if (res.insertErrorInNode(nodes[1], 7192))
2395  return NDBT_FAILED;
2396 
2397  if (res.insertErrorInNode(nodes[0], 7191))
2398  return NDBT_FAILED;
2399 
2400  if (res.waitClusterNoStart())
2401  return NDBT_FAILED;
2402 
2403  if (res.startAll())
2404  return NDBT_FAILED;
2405 
2406  if (res.waitClusterStarted())
2407  return NDBT_FAILED;
2408 
2409  if (res.restartOneDbNode(nodes[1], false, false, true))
2410  return NDBT_FAILED;
2411 
2412  if (res.waitClusterStarted())
2413  return NDBT_FAILED;
2414 
2415  return NDBT_OK;
2416 }
2417 
2418 int
2419 runBug31980(NDBT_Context* ctx, NDBT_Step* step)
2420 {
2421  //int result = NDBT_OK;
2422  //int loops = ctx->getNumLoops();
2423  //int records = ctx->getNumRecords();
2424  Ndb* pNdb = GETNDB(step);
2425  NdbRestarter res;
2426 
2427  if (res.getNumDbNodes() < 2)
2428  {
2429  return NDBT_OK;
2430  }
2431 
2432 
2433  HugoOperations hugoOps (* ctx->getTab());
2434  if(hugoOps.startTransaction(pNdb) != 0)
2435  return NDBT_FAILED;
2436 
2437  if(hugoOps.pkInsertRecord(pNdb, 1) != 0)
2438  return NDBT_FAILED;
2439 
2440  if(hugoOps.execute_NoCommit(pNdb) != 0)
2441  return NDBT_FAILED;
2442 
2443  int transNode= hugoOps.getTransaction()->getConnectedNodeId();
2444  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2445 
2446  if (res.dumpStateOneNode(transNode, val2, 2))
2447  {
2448  return NDBT_FAILED;
2449  }
2450 
2451  if (res.insertErrorInNode(transNode, 8055))
2452  {
2453  return NDBT_FAILED;
2454  }
2455 
2456  hugoOps.execute_Commit(pNdb); // This should hang/fail
2457 
2458  if (res.waitNodesNoStart(&transNode, 1))
2459  return NDBT_FAILED;
2460 
2461  if (res.startNodes(&transNode, 1))
2462  return NDBT_FAILED;
2463 
2464  if (res.waitClusterStarted())
2465  return NDBT_FAILED;
2466 
2467  return NDBT_OK;
2468 }
2469 
2470 int
2471 runBug32160(NDBT_Context* ctx, NDBT_Step* step)
2472 {
2473  //int result = NDBT_OK;
2474  //int loops = ctx->getNumLoops();
2475  //int records = ctx->getNumRecords();
2476  //Ndb* pNdb = GETNDB(step);
2477  NdbRestarter res;
2478 
2479  if (res.getNumDbNodes() < 2)
2480  {
2481  return NDBT_OK;
2482  }
2483 
2484  int master = res.getMasterNodeId();
2485  int next = res.getNextMasterNodeId(master);
2486 
2487  if (res.insertErrorInNode(next, 7194))
2488  {
2489  return NDBT_FAILED;
2490  }
2491 
2492  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2493  if (res.dumpStateOneNode(master, val2, 2))
2494  return NDBT_FAILED;
2495 
2496  if (res.insertErrorInNode(master, 7193))
2497  return NDBT_FAILED;
2498 
2499  int val3[] = { 7099 };
2500  if (res.dumpStateOneNode(master, val3, 1))
2501  return NDBT_FAILED;
2502 
2503  if (res.waitNodesNoStart(&master, 1))
2504  return NDBT_FAILED;
2505 
2506  if (res.startNodes(&master, 1))
2507  return NDBT_FAILED;
2508 
2509  if (res.waitClusterStarted())
2510  return NDBT_FAILED;
2511 
2512  return NDBT_OK;
2513 }
2514 
2515 int
2516 runBug32922(NDBT_Context* ctx, NDBT_Step* step)
2517 {
2518  //int result = NDBT_OK;
2519  int loops = ctx->getNumLoops();
2520  //int records = ctx->getNumRecords();
2521  //Ndb* pNdb = GETNDB(step);
2522  NdbRestarter res;
2523 
2524  if (res.getNumDbNodes() < 2)
2525  {
2526  return NDBT_OK;
2527  }
2528 
2529  while (loops--)
2530  {
2531  int master = res.getMasterNodeId();
2532 
2533  int victim = 32768;
2534  for (Uint32 i = 0; i<(Uint32)res.getNumDbNodes(); i++)
2535  {
2536  int node = res.getDbNodeId(i);
2537  if (node != master && node < victim)
2538  victim = node;
2539  }
2540 
2541  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2542  if (res.dumpStateOneNode(victim, val2, 2))
2543  return NDBT_FAILED;
2544 
2545  if (res.insertErrorInNode(master, 7200))
2546  return NDBT_FAILED;
2547 
2548  if (res.waitNodesNoStart(&victim, 1))
2549  return NDBT_FAILED;
2550 
2551  if (res.startNodes(&victim, 1))
2552  return NDBT_FAILED;
2553 
2554  if (res.waitClusterStarted())
2555  return NDBT_FAILED;
2556  }
2557 
2558  return NDBT_OK;
2559 }
2560 
2561 int
2562 runBug34216(NDBT_Context* ctx, NDBT_Step* step)
2563 {
2564  int result = NDBT_OK;
2565  int loops = ctx->getNumLoops();
2566  NdbRestarter restarter;
2567  int i = 0;
2568  int lastId = 0;
2569  HugoOperations hugoOps(*ctx->getTab());
2570  int records = ctx->getNumRecords();
2571  Ndb* pNdb = GETNDB(step);
2572 
2573  if (restarter.getNumDbNodes() < 2)
2574  {
2575  ctx->stopTest();
2576  return NDBT_OK;
2577  }
2578 
2579  if(restarter.waitClusterStarted() != 0){
2580  g_err << "Cluster failed to start" << endl;
2581  return NDBT_FAILED;
2582  }
2583 
2584  char buf[100];
2585  const char * off = NdbEnv_GetEnv("NDB_ERR_OFFSET", buf, sizeof(buf));
2586  int offset = off ? atoi(off) : 0;
2587 
2588  while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped())
2589  {
2590  int id = lastId % restarter.getNumDbNodes();
2591  int nodeId = restarter.getDbNodeId(id);
2592  int err = 5048 + ((i+offset) % 2);
2593 
2594  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2595 
2596  if(hugoOps.startTransaction(pNdb) != 0)
2597  goto err;
2598 
2599  nodeId = hugoOps.getTransaction()->getConnectedNodeId();
2600  ndbout << "Restart node " << nodeId << " " << err <<endl;
2601 
2602  if (restarter.dumpStateOneNode(nodeId, val2, 2))
2603  return NDBT_FAILED;
2604 
2605  if(restarter.insertErrorInNode(nodeId, err) != 0){
2606  g_err << "Failed to restartNextDbNode" << endl;
2607  result = NDBT_FAILED;
2608  break;
2609  }
2610 
2611  if (restarter.insertErrorInNode(nodeId, 8057) != 0)
2612  {
2613  g_err << "Failed to insert error 8057" << endl;
2614  result = NDBT_FAILED;
2615  break;
2616  }
2617 
2618  int rows = 25;
2619  if (rows > records)
2620  rows = records;
2621 
2622  int batch = 1;
2623  int row = (records - rows) ? rand() % (records - rows) : 0;
2624  if (row + rows > records)
2625  row = records - row;
2626 
2631  for (int r = row; r < row + rows; r++)
2632  {
2633  if(hugoOps.pkUpdateRecord(pNdb, r, batch, rand()) != 0)
2634  goto err;
2635 
2636  for (int l = 1; l<5; l++)
2637  {
2638  if (hugoOps.execute_NoCommit(pNdb) != 0)
2639  goto err;
2640 
2641  if(hugoOps.pkUpdateRecord(pNdb, r, batch, rand()) != 0)
2642  goto err;
2643  }
2644  }
2645 
2646  hugoOps.execute_Commit(pNdb);
2647  hugoOps.closeTransaction(pNdb);
2648 
2649  if (restarter.waitNodesNoStart(&nodeId, 1))
2650  {
2651  g_err << "Failed to waitNodesNoStart" << endl;
2652  result = NDBT_FAILED;
2653  break;
2654  }
2655 
2656  if (restarter.startNodes(&nodeId, 1))
2657  {
2658  g_err << "Failed to startNodes" << endl;
2659  result = NDBT_FAILED;
2660  break;
2661  }
2662 
2663  if(restarter.waitClusterStarted() != 0){
2664  g_err << "Cluster failed to start" << endl;
2665  result = NDBT_FAILED;
2666  break;
2667  }
2668 
2669  lastId++;
2670  i++;
2671  }
2672 
2673  ctx->stopTest();
2674 
2675  return result;
2676 err:
2677  return NDBT_FAILED;
2678 }
2679 
2680 
2681 int
2682 runNF_commit(NDBT_Context* ctx, NDBT_Step* step)
2683 {
2684  int result = NDBT_OK;
2685  int loops = ctx->getNumLoops();
2686  NdbRestarter restarter;
2687  if (restarter.getNumDbNodes() < 2)
2688  {
2689  ctx->stopTest();
2690  return NDBT_OK;
2691  }
2692 
2693  if(restarter.waitClusterStarted() != 0){
2694  g_err << "Cluster failed to start" << endl;
2695  return NDBT_FAILED;
2696  }
2697 
2698  int i = 0;
2699  while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped())
2700  {
2701  int nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
2702  int err = 5048;
2703 
2704  ndbout << "Restart node " << nodeId << " " << err <<endl;
2705 
2706  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2707  if (restarter.dumpStateOneNode(nodeId, val2, 2))
2708  return NDBT_FAILED;
2709 
2710  if(restarter.insertErrorInNode(nodeId, err) != 0){
2711  g_err << "Failed to restartNextDbNode" << endl;
2712  result = NDBT_FAILED;
2713  break;
2714  }
2715 
2716  if (restarter.waitNodesNoStart(&nodeId, 1))
2717  {
2718  g_err << "Failed to waitNodesNoStart" << endl;
2719  result = NDBT_FAILED;
2720  break;
2721  }
2722 
2723  if (restarter.startNodes(&nodeId, 1))
2724  {
2725  g_err << "Failed to startNodes" << endl;
2726  result = NDBT_FAILED;
2727  break;
2728  }
2729 
2730  if(restarter.waitClusterStarted() != 0){
2731  g_err << "Cluster failed to start" << endl;
2732  result = NDBT_FAILED;
2733  break;
2734  }
2735 
2736  i++;
2737  }
2738 
2739  ctx->stopTest();
2740 
2741  return result;
2742 }
2743 
2744 int
2745 runBug34702(NDBT_Context* ctx, NDBT_Step* step)
2746 {
2747  //int result = NDBT_OK;
2748  int loops = ctx->getNumLoops();
2749  //int records = ctx->getNumRecords();
2750  //Ndb* pNdb = GETNDB(step);
2751  NdbRestarter res;
2752 
2753  if (res.getNumDbNodes() < 2)
2754  {
2755  return NDBT_OK;
2756  }
2757 
2758  while (loops--)
2759  {
2760  int victim = res.getDbNodeId(rand()%res.getNumDbNodes());
2761  res.restartOneDbNode(victim, true, true, true);
2765 
2766  if (res.waitNodesNoStart(&victim, 1))
2767  return NDBT_FAILED;
2768 
2769  res.insertErrorInAllNodes(7204);
2770  res.insertErrorInNode(victim, 7203);
2771 
2772  res.startNodes(&victim, 1);
2773 
2774  if (res.waitClusterStarted())
2775  return NDBT_FAILED;
2776  }
2777  return NDBT_OK;
2778 }
2779 
2780 int
2781 runMNF(NDBT_Context* ctx, NDBT_Step* step)
2782 {
2783  //int result = NDBT_OK;
2784  NdbRestarter res;
2785 
2786  if (res.getNumDbNodes() < 2)
2787  {
2788  return NDBT_OK;
2789  }
2790 
2791  Vector<int> part0;
2792  Vector<int> part1;
2793  Bitmask<255> part0mask;
2794  Bitmask<255> part1mask;
2795  Bitmask<255> ngmask;
2796  for (int i = 0; i<res.getNumDbNodes(); i++)
2797  {
2798  int nodeId = res.getDbNodeId(i);
2799  int ng = res.getNodeGroup(nodeId);
2800  if (ngmask.get(ng))
2801  {
2802  part1.push_back(nodeId);
2803  part1mask.set(nodeId);
2804  }
2805  else
2806  {
2807  ngmask.set(ng);
2808  part0.push_back(nodeId);
2809  part0mask.set(nodeId);
2810  }
2811  }
2812 
2813  printf("part0: ");
2814  for (size_t i = 0; i<part0.size(); i++)
2815  printf("%u ", part0[i]);
2816  printf("\n");
2817 
2818  printf("part1: ");
2819  for (size_t i = 0; i<part1.size(); i++)
2820  printf("%u ", part1[i]);
2821  printf("\n");
2822 
2823  int loops = ctx->getNumLoops();
2824  while (loops-- && !ctx->isTestStopped())
2825  {
2826  int cnt, *nodes;
2827  int master = res.getMasterNodeId();
2828  int nextMaster = res.getNextMasterNodeId(master);
2829 
2830  bool cmf = false;
2831  if (part0mask.get(master) && part0mask.get(nextMaster))
2832  {
2833  cmf = true;
2834  cnt = part0.size();
2835  nodes = part0.getBase();
2836  printf("restarting part0");
2837  }
2838  else if(part1mask.get(master) && part1mask.get(nextMaster))
2839  {
2840  cmf = true;
2841  cnt = part1.size();
2842  nodes = part1.getBase();
2843  printf("restarting part1");
2844  }
2845  else
2846  {
2847  cmf = false;
2848  if (loops & 1)
2849  {
2850  cnt = part0.size();
2851  nodes = part0.getBase();
2852  printf("restarting part0");
2853  }
2854  else
2855  {
2856  cnt = part1.size();
2857  nodes = part0.getBase();
2858  printf("restarting part0");
2859  }
2860  }
2861 
2862  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2863  for (int i = 0; i<cnt; i++)
2864  if (res.dumpStateOneNode(nodes[i], val2, 2))
2865  return NDBT_FAILED;
2866 
2867  int type = loops;
2868  char buf[100];
2869  if (NdbEnv_GetEnv("MNF", buf, sizeof(buf)))
2870  {
2871  type = atoi(buf);
2872  }
2873  if (cmf)
2874  {
2875  type = type % 7;
2876  }
2877  else
2878  {
2879  type = type % 4;
2880  }
2881  ndbout_c(" type: %u (cmf: %u)", type, cmf);
2882  switch(type){
2883  case 0:
2884  for (int i = 0; i<cnt; i++)
2885  {
2886  if (res.restartOneDbNode(nodes[i], false, true, true))
2890  return NDBT_FAILED;
2891 
2892  NdbSleep_MilliSleep(10);
2893  }
2894  break;
2895  case 1:
2896  for (int i = 0; i<cnt; i++)
2897  {
2898  if (res.restartOneDbNode(nodes[i], false, true, true))
2902  return NDBT_FAILED;
2903 
2904  }
2905  break;
2906  case 2:
2907  for (int i = 0; i<cnt; i++)
2908  {
2909  res.insertErrorInNode(nodes[i], 8058);
2910  }
2911  res.restartOneDbNode(nodes[0], false, true, true);
2915  break;
2916  case 3:
2917  for (int i = 0; i<cnt; i++)
2918  {
2919  res.insertErrorInNode(nodes[i], 8059);
2920  }
2921  res.restartOneDbNode(nodes[0], false, true, true);
2925  break;
2926  case 4:
2927  {
2928  for (int i = 0; i<cnt; i++)
2929  {
2930  if (res.getNextMasterNodeId(master) == nodes[i])
2931  res.insertErrorInNode(nodes[i], 7180);
2932  else
2933  res.insertErrorInNode(nodes[i], 7205);
2934  }
2935 
2936  int lcp = 7099;
2937  res.insertErrorInNode(master, 7193);
2938  res.dumpStateOneNode(master, &lcp, 1);
2939  break;
2940  }
2941  case 5:
2942  {
2943  for (int i = 0; i<cnt; i++)
2944  {
2945  res.insertErrorInNode(nodes[i], 7206);
2946  }
2947 
2948  int lcp = 7099;
2949  res.insertErrorInNode(master, 7193);
2950  res.dumpStateOneNode(master, &lcp, 1);
2951  break;
2952  }
2953  case 6:
2954  {
2955  for (int i = 0; i<cnt; i++)
2956  {
2957  res.insertErrorInNode(nodes[i], 5008);
2958  }
2959 
2960  int lcp = 7099;
2961  res.insertErrorInNode(master, 7193);
2962  res.dumpStateOneNode(master, &lcp, 1);
2963  break;
2964  }
2965  }
2966 
2967  if (res.waitNodesNoStart(nodes, cnt))
2968  return NDBT_FAILED;
2969 
2970  if (res.startNodes(nodes, cnt))
2971  return NDBT_FAILED;
2972 
2973  if (res.waitClusterStarted())
2974  return NDBT_FAILED;
2975  }
2976 
2977  ctx->stopTest();
2978  return NDBT_OK;
2979 }
2980 
2981 int
2982 runBug36199(NDBT_Context* ctx, NDBT_Step* step)
2983 {
2984  //int result = NDBT_OK;
2985  //int loops = ctx->getNumLoops();
2986  NdbRestarter res;
2987 
2988  if (res.getNumDbNodes() < 4)
2989  return NDBT_OK;
2990 
2991  int master = res.getMasterNodeId();
2992  int nextMaster = res.getNextMasterNodeId(master);
2993  int victim = res.getRandomNodeSameNodeGroup(nextMaster, rand());
2994  if (victim == master)
2995  {
2996  victim = res.getRandomNodeOtherNodeGroup(nextMaster, rand());
2997  }
2998 
2999  ndbout_c("master: %u next master: %u victim: %u",
3000  master, nextMaster, victim);
3001 
3002  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3003  res.dumpStateOneNode(master, val2, 2);
3004  res.dumpStateOneNode(victim, val2, 2);
3005 
3006  res.insertErrorInNode(victim, 7205);
3007  res.insertErrorInNode(master, 7014);
3008  int lcp = 7099;
3009  res.dumpStateOneNode(master, &lcp, 1);
3010 
3011  int nodes[2];
3012  nodes[0] = master;
3013  nodes[1] = victim;
3014  if (res.waitNodesNoStart(nodes, 2))
3015  {
3016  return NDBT_FAILED;
3017  }
3018 
3019  if (res.startNodes(nodes, 2))
3020  {
3021  return NDBT_FAILED;
3022  }
3023 
3024  if (res.waitClusterStarted())
3025  return NDBT_FAILED;
3026 
3027  return NDBT_OK;
3028 }
3029 
3030 int
3031 runBug36246(NDBT_Context* ctx, NDBT_Step* step)
3032 {
3033  //int result = NDBT_OK;
3034  //int loops = ctx->getNumLoops();
3035  NdbRestarter res;
3036  Ndb* pNdb = GETNDB(step);
3037 
3038  if (res.getNumDbNodes() < 4)
3039  return NDBT_OK;
3040 
3041  HugoOperations hugoOps(*ctx->getTab());
3042 restartloop:
3043  int tryloop = 0;
3044  int master = res.getMasterNodeId();
3045  int nextMaster = res.getNextMasterNodeId(master);
3046 
3047 loop:
3048  if(hugoOps.startTransaction(pNdb) != 0)
3049  return NDBT_FAILED;
3050 
3051  if(hugoOps.pkUpdateRecord(pNdb, 1, 1) != 0)
3052  return NDBT_FAILED;
3053 
3054  if(hugoOps.execute_NoCommit(pNdb) != 0)
3055  return NDBT_FAILED;
3056 
3057  int victim = hugoOps.getTransaction()->getConnectedNodeId();
3058  printf("master: %u nextMaster: %u victim: %u",
3059  master, nextMaster, victim);
3060  if (victim == master || victim == nextMaster ||
3061  res.getNodeGroup(victim) == res.getNodeGroup(master) ||
3062  res.getNodeGroup(victim) == res.getNodeGroup(nextMaster))
3063  {
3064  hugoOps.execute_Rollback(pNdb);
3065  hugoOps.closeTransaction(pNdb);
3066  tryloop++;
3067  if (tryloop == 10)
3068  {
3069  ndbout_c(" -> restarting next master: %u", nextMaster);
3070  res.restartOneDbNode(nextMaster, false, true, true);
3074 
3075  res.waitNodesNoStart(&nextMaster, 1);
3076  res.startNodes(&nextMaster, 1);
3077  if (res.waitClusterStarted())
3078  return NDBT_FAILED;
3079  goto restartloop;
3080  }
3081  else
3082  {
3083  ndbout_c(" -> loop");
3084  goto loop;
3085  }
3086  }
3087  ndbout_c(" -> go go gadget skates");
3088 
3089  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3090  res.dumpStateOneNode(master, val2, 2);
3091  res.dumpStateOneNode(victim, val2, 2);
3092 
3093  res.insertErrorInNode(master, 8060);
3094  res.insertErrorInNode(victim, 9999);
3095 
3096  int nodes[2];
3097  nodes[0] = master;
3098  nodes[1] = victim;
3099  if (res.waitNodesNoStart(nodes, 2))
3100  {
3101  return NDBT_FAILED;
3102  }
3103 
3104  if (res.startNodes(nodes, 2))
3105  {
3106  return NDBT_FAILED;
3107  }
3108 
3109  if (res.waitClusterStarted())
3110  return NDBT_FAILED;
3111 
3112  hugoOps.execute_Rollback(pNdb);
3113  hugoOps.closeTransaction(pNdb);
3114 
3115  return NDBT_OK;
3116 }
3117 
3118 int
3119 runBug36247(NDBT_Context* ctx, NDBT_Step* step)
3120 {
3121  //int result = NDBT_OK;
3122  //int loops = ctx->getNumLoops();
3123  NdbRestarter res;
3124  Ndb* pNdb = GETNDB(step);
3125 
3126  if (res.getNumDbNodes() < 4)
3127  return NDBT_OK;
3128 
3129  HugoOperations hugoOps(*ctx->getTab());
3130 
3131 restartloop:
3132  int tryloop = 0;
3133  int master = res.getMasterNodeId();
3134  int nextMaster = res.getNextMasterNodeId(master);
3135 
3136 loop:
3137  if(hugoOps.startTransaction(pNdb) != 0)
3138  return NDBT_FAILED;
3139 
3140  if(hugoOps.pkUpdateRecord(pNdb, 1, 100) != 0)
3141  return NDBT_FAILED;
3142 
3143  if(hugoOps.execute_NoCommit(pNdb) != 0)
3144  return NDBT_FAILED;
3145 
3146  int victim = hugoOps.getTransaction()->getConnectedNodeId();
3147  printf("master: %u nextMaster: %u victim: %u",
3148  master, nextMaster, victim);
3149  if (victim == master || victim == nextMaster ||
3150  res.getNodeGroup(victim) == res.getNodeGroup(master) ||
3151  res.getNodeGroup(victim) == res.getNodeGroup(nextMaster))
3152  {
3153  hugoOps.execute_Rollback(pNdb);
3154  hugoOps.closeTransaction(pNdb);
3155  tryloop++;
3156  if (tryloop == 10)
3157  {
3158  ndbout_c(" -> restarting next master: %u", nextMaster);
3159  res.restartOneDbNode(nextMaster, false, true, true);
3163 
3164  res.waitNodesNoStart(&nextMaster, 1);
3165  res.startNodes(&nextMaster, 1);
3166  if (res.waitClusterStarted())
3167  return NDBT_FAILED;
3168  goto restartloop;
3169  }
3170  else
3171  {
3172  ndbout_c(" -> loop");
3173  goto loop;
3174  }
3175  }
3176  ndbout_c(" -> go go gadget skates");
3177 
3178  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3179  res.dumpStateOneNode(master, val2, 2);
3180  res.dumpStateOneNode(victim, val2, 2);
3181 
3182  for (int i = 0; i<res.getNumDbNodes(); i++)
3183  {
3184  int nodeId = res.getDbNodeId(i);
3185  res.insertErrorInNode(nodeId, 5050);
3186  }
3187 
3188  res.insertErrorInNode(victim, 9999);
3189 
3190  int nodes[2];
3191  nodes[0] = master;
3192  nodes[1] = victim;
3193  if (res.waitNodesNoStart(nodes, 2))
3194  {
3195  return NDBT_FAILED;
3196  }
3197 
3198  if (res.startNodes(nodes, 2))
3199  {
3200  return NDBT_FAILED;
3201  }
3202 
3203  if (res.waitClusterStarted())
3204  return NDBT_FAILED;
3205 
3206  hugoOps.execute_Rollback(pNdb);
3207  hugoOps.closeTransaction(pNdb);
3208 
3209  return NDBT_OK;
3210 }
3211 
3212 int
3213 runBug36276(NDBT_Context* ctx, NDBT_Step* step)
3214 {
3215  //int result = NDBT_OK;
3216  //int loops = ctx->getNumLoops();
3217  NdbRestarter res;
3218  //Ndb* pNdb = GETNDB(step);
3219 
3220  if (res.getNumDbNodes() < 4)
3221  return NDBT_OK;
3222 
3223  int master = res.getMasterNodeId();
3224  int nextMaster = res.getNextMasterNodeId(master);
3225  int victim = res.getRandomNodeSameNodeGroup(nextMaster, rand());
3226  if (victim == master)
3227  {
3228  victim = res.getRandomNodeOtherNodeGroup(nextMaster, rand());
3229  }
3230 
3231  ndbout_c("master: %u nextMaster: %u victim: %u",
3232  master, nextMaster, victim);
3233 
3234  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3235  res.dumpStateOneNode(master, val2, 2);
3236  res.insertErrorInNode(victim, 7209);
3237 
3238  int lcp = 7099;
3239  res.dumpStateOneNode(master, &lcp, 1);
3240 
3241  if (res.waitNodesNoStart(&master, 1))
3242  {
3243  return NDBT_FAILED;
3244  }
3245 
3246  if (res.startNodes(&master, 1))
3247  {
3248  return NDBT_FAILED;
3249  }
3250 
3251  if (res.waitClusterStarted())
3252  return NDBT_FAILED;
3253 
3254  return NDBT_OK;
3255 }
3256 
3257 int
3258 runBug36245(NDBT_Context* ctx, NDBT_Step* step)
3259 {
3260  //int result = NDBT_OK;
3261  //int loops = ctx->getNumLoops();
3262  NdbRestarter res;
3263  Ndb* pNdb = GETNDB(step);
3264 
3265  if (res.getNumDbNodes() < 4)
3266  return NDBT_OK;
3267 
3271 loop1:
3272  int master = res.getMasterNodeId();
3273  int nextMaster = res.getNextMasterNodeId(master);
3274 
3275  printf("master: %u nextMaster: %u", master, nextMaster);
3276  if (res.getNodeGroup(master) == res.getNodeGroup(nextMaster))
3277  {
3278  ndbout_c(" -> restarting next master: %u", nextMaster);
3279  res.restartOneDbNode(nextMaster, false, true, true);
3283 
3284  res.waitNodesNoStart(&nextMaster, 1);
3285  res.startNodes(&nextMaster, 1);
3286  if (res.waitClusterStarted())
3287  {
3288  ndbout_c("cluster didnt restart!!");
3289  return NDBT_FAILED;
3290  }
3291  goto loop1;
3292  }
3293  ndbout_c(" -> go go gadget skates");
3294 
3295  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3296  res.dumpStateOneNode(master, val2, 2);
3297  res.dumpStateOneNode(nextMaster, val2, 2);
3298 
3299  res.insertErrorInNode(master, 8063);
3300  res.insertErrorInNode(nextMaster, 936);
3301 
3302 
3303  int err = 0;
3304  HugoOperations hugoOps(*ctx->getTab());
3305 loop2:
3306  if((err = hugoOps.startTransaction(pNdb)) != 0)
3307  {
3308  ndbout_c("failed to start transaction: %u", err);
3309  return NDBT_FAILED;
3310  }
3311 
3312  int victim = hugoOps.getTransaction()->getConnectedNodeId();
3313  if (victim != master)
3314  {
3315  ndbout_c("transnode: %u != master: %u -> loop",
3316  victim, master);
3317  hugoOps.closeTransaction(pNdb);
3318  goto loop2;
3319  }
3320 
3321  if((err = hugoOps.pkUpdateRecord(pNdb, 1)) != 0)
3322  {
3323  ndbout_c("failed to update: %u", err);
3324  return NDBT_FAILED;
3325  }
3326 
3327  if((err = hugoOps.execute_Commit(pNdb)) != 4010)
3328  {
3329  ndbout_c("incorrect error code: %u", err);
3330  return NDBT_FAILED;
3331  }
3332  hugoOps.closeTransaction(pNdb);
3333 
3334  int nodes[2];
3335  nodes[0] = master;
3336  nodes[1] = nextMaster;
3337  if (res.waitNodesNoStart(nodes, 2))
3338  {
3339  return NDBT_FAILED;
3340  }
3341 
3342  if (res.startNodes(nodes, 2))
3343  {
3344  return NDBT_FAILED;
3345  }
3346 
3347  if (res.waitClusterStarted())
3348  return NDBT_FAILED;
3349 
3350  return NDBT_OK;
3351 }
3352 
3353 int
3354 runHammer(NDBT_Context* ctx, NDBT_Step* step)
3355 {
3356  int records = ctx->getNumRecords();
3357  Ndb* pNdb = GETNDB(step);
3358  HugoOperations hugoOps(*ctx->getTab());
3359  while (!ctx->isTestStopped())
3360  {
3361  int r = rand() % records;
3362  if (hugoOps.startTransaction(pNdb) != 0)
3363  continue;
3364 
3365  if ((rand() % 100) < 50)
3366  {
3367  if (hugoOps.pkUpdateRecord(pNdb, r, 1, rand()) != 0)
3368  goto err;
3369  }
3370  else
3371  {
3372  if (hugoOps.pkWriteRecord(pNdb, r, 1, rand()) != 0)
3373  goto err;
3374  }
3375 
3376  if (hugoOps.execute_NoCommit(pNdb) != 0)
3377  goto err;
3378 
3379  if (hugoOps.pkDeleteRecord(pNdb, r, 1) != 0)
3380  goto err;
3381 
3382  if (hugoOps.execute_NoCommit(pNdb) != 0)
3383  goto err;
3384 
3385  if ((rand() % 100) < 50)
3386  {
3387  if (hugoOps.pkInsertRecord(pNdb, r, 1, rand()) != 0)
3388  goto err;
3389  }
3390  else
3391  {
3392  if (hugoOps.pkWriteRecord(pNdb, r, 1, rand()) != 0)
3393  goto err;
3394  }
3395 
3396  if ((rand() % 100) < 90)
3397  {
3398  hugoOps.execute_Commit(pNdb);
3399  }
3400  else
3401  {
3402  err:
3403  hugoOps.execute_Rollback(pNdb);
3404  }
3405 
3406  hugoOps.closeTransaction(pNdb);
3407  }
3408  return NDBT_OK;
3409 }
3410 
3411 int
3412 runMixedLoad(NDBT_Context* ctx, NDBT_Step* step)
3413 {
3414  int res = 0;
3415  int records = ctx->getNumRecords();
3416  Ndb* pNdb = GETNDB(step);
3417  HugoOperations hugoOps(*ctx->getTab());
3418  unsigned id = (unsigned)rand();
3419  while (!ctx->isTestStopped())
3420  {
3421  if (ctx->getProperty("Pause", (Uint32)0))
3422  {
3423  ndbout_c("thread %u stopped", id);
3424  ctx->sync_down("WaitThreads");
3425  while (ctx->getProperty("Pause", (Uint32)0) && !ctx->isTestStopped())
3426  NdbSleep_MilliSleep(15);
3427 
3428  if (ctx->isTestStopped())
3429  break;
3430  ndbout_c("thread %u continue", id);
3431  }
3432 
3433  if ((res = hugoOps.startTransaction(pNdb)) != 0)
3434  {
3435  if (res == 4009)
3436  return NDBT_FAILED;
3437  continue;
3438  }
3439 
3440  for (int i = 0; i < 10; i++)
3441  {
3442  int r = rand() % records;
3443  if ((rand() % 100) < 50)
3444  {
3445  if (hugoOps.pkUpdateRecord(pNdb, r, 1, rand()) != 0)
3446  goto err;
3447  }
3448  else
3449  {
3450  if (hugoOps.pkWriteRecord(pNdb, r, 1, rand()) != 0)
3451  goto err;
3452  }
3453  }
3454 
3455  if ((rand() % 100) < 90)
3456  {
3457  res = hugoOps.execute_Commit(pNdb);
3458  }
3459  else
3460  {
3461  err:
3462  res = hugoOps.execute_Rollback(pNdb);
3463  }
3464 
3465  hugoOps.closeTransaction(pNdb);
3466 
3467  if (res == 4009)
3468  {
3469  return NDBT_FAILED;
3470  }
3471  }
3472  return NDBT_OK;
3473 }
3474 
3475 int
3476 runBug41295(NDBT_Context* ctx, NDBT_Step* step)
3477 {
3478  NdbRestarter res;
3479 
3480  if (res.getNumDbNodes() < 2)
3481  {
3482  ctx->stopTest();
3483  return NDBT_OK;
3484  }
3485 
3486 
3487  int leak = 4002;
3488  const int cases = 1;
3489  int loops = ctx->getNumLoops();
3490  if (loops <= cases)
3491  loops = cases + 1;
3492 
3493  for (int i = 0; i<loops; i++)
3494  {
3495  int master = res.getMasterNodeId();
3496  int next = res.getNextMasterNodeId(master);
3497 
3498  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3499  if (res.dumpStateOneNode(next, val2, 2))
3500  return NDBT_FAILED;
3501 
3502  ndbout_c("stopping %u, err 8073", next);
3503  res.insertErrorInNode(next, 8073);
3504  ndbout_c("waiting for %u", next);
3505  res.waitNodesNoStart(&next, 1);
3506 
3507  ndbout_c("pausing all threads");
3508  ctx->setProperty("Pause", 1);
3509  ctx->sync_up_and_wait("WaitThreads", ctx->getProperty("Threads", 1));
3510  ndbout_c("all threads paused");
3511  NdbSleep_MilliSleep(5000);
3512  res.dumpStateAllNodes(&leak, 1);
3513  NdbSleep_MilliSleep(1000);
3514  if (res.checkClusterAlive(&next, 1))
3515  {
3516  return NDBT_FAILED;
3517  }
3518  ndbout_c("restarting threads");
3519  ctx->setProperty("Pause", (Uint32)0);
3520 
3521  ndbout_c("starting %u", next);
3522  res.startNodes(&next, 1);
3523  ndbout_c("waiting for cluster started");
3524  if (res.waitClusterStarted())
3525  {
3526  return NDBT_FAILED;
3527  }
3528 
3529  ndbout_c("pausing all threads");
3530  ctx->setProperty("Pause", 1);
3531  ctx->sync_up_and_wait("WaitThreads", ctx->getProperty("Threads", 1));
3532  ndbout_c("all threads paused");
3533  NdbSleep_MilliSleep(5000);
3534  res.dumpStateAllNodes(&leak, 1);
3535  NdbSleep_MilliSleep(1000);
3536  ndbout_c("restarting threads");
3537  ctx->setProperty("Pause", (Uint32)0);
3538  }
3539 
3540  ctx->stopTest();
3541  return NDBT_OK;
3542 }
3543 
3544 int
3545 runBug41469(NDBT_Context* ctx, NDBT_Step* step)
3546 {
3547  NdbRestarter res;
3548 
3549  if (res.getNumDbNodes() < 4)
3550  {
3551  ctx->stopTest();
3552  return NDBT_OK;
3553  }
3554 
3555  int loops = ctx->getNumLoops();
3556 
3557  int val0[] = { 7216, 0 };
3558  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3559  for (int i = 0; i<loops; i++)
3560  {
3561  int master = res.getMasterNodeId();
3562  int next = res.getNextMasterNodeId(master);
3563 
3564  if (res.dumpStateOneNode(master, val2, 2))
3565  return NDBT_FAILED;
3566 
3567  ndbout_c("stopping %u, err 7216 (next: %u)", master, next);
3568  val0[1] = next;
3569  if (res.dumpStateOneNode(master, val0, 2))
3570  return NDBT_FAILED;
3571 
3572  res.waitNodesNoStart(&master, 1);
3573  res.startNodes(&master, 1);
3574  ndbout_c("waiting for cluster started");
3575  if (res.waitClusterStarted())
3576  {
3577  return NDBT_FAILED;
3578  }
3579  }
3580  ctx->stopTest();
3581  return NDBT_OK;
3582 }
3583 
3584 int
3585 runBug42422(NDBT_Context* ctx, NDBT_Step* step)
3586 {
3587  NdbRestarter res;
3588 
3589  if (res.getNumDbNodes() < 4)
3590  {
3591  ctx->stopTest();
3592  return NDBT_OK;
3593  }
3594 
3595  int loops = ctx->getNumLoops();
3596  while (--loops >= 0)
3597  {
3598  int master = res.getMasterNodeId();
3599  ndbout_c("master: %u", master);
3600  int nodeId = res.getRandomNodeSameNodeGroup(master, rand());
3601  ndbout_c("target: %u", nodeId);
3602  int node2 = res.getRandomNodeOtherNodeGroup(nodeId, rand());
3603  ndbout_c("node 2: %u", node2);
3604 
3605  res.restartOneDbNode(nodeId, false, true, true);
3609 
3610  res.waitNodesNoStart(&nodeId, 1);
3611 
3612  int dump[] = { 9000, 0 };
3613  dump[1] = node2;
3614 
3615  if (res.dumpStateOneNode(nodeId, dump, 2))
3616  return NDBT_FAILED;
3617 
3618  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3619  if (res.dumpStateOneNode(nodeId, val2, 2))
3620  return NDBT_FAILED;
3621 
3622  res.insertErrorInNode(nodeId, 937);
3623  ndbout_c("%u : starting %u", __LINE__, nodeId);
3624  res.startNodes(&nodeId, 1);
3625  NdbSleep_SecSleep(3);
3626  ndbout_c("%u : waiting for %u to not get not-started", __LINE__, nodeId);
3627  res.waitNodesNoStart(&nodeId, 1);
3628 
3629  ndbout_c("%u : starting %u", __LINE__, nodeId);
3630  res.startNodes(&nodeId, 1);
3631 
3632  ndbout_c("%u : waiting for cluster started", __LINE__);
3633  if (res.waitClusterStarted())
3634  {
3635  return NDBT_FAILED;
3636  }
3637  }
3638 
3639  ctx->stopTest();
3640  return NDBT_OK;
3641 }
3642 
3643 int
3644 runBug43224(NDBT_Context* ctx, NDBT_Step* step)
3645 {
3646  NdbRestarter res;
3647 
3648  if (res.getNumDbNodes() < 2)
3649  {
3650  ctx->stopTest();
3651  return NDBT_OK;
3652  }
3653 
3654  int loops = ctx->getNumLoops();
3655  while (--loops >= 0)
3656  {
3657  int nodeId = res.getNode(NdbRestarter::NS_RANDOM);
3658  res.restartOneDbNode(nodeId, false, true, true);
3662 
3663  res.waitNodesNoStart(&nodeId, 1);
3664 
3665  NdbSleep_SecSleep(10);
3666 
3667  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3668  if (res.dumpStateOneNode(nodeId, val2, 2))
3669  return NDBT_FAILED;
3670 
3671  res.insertErrorInNode(nodeId, 9994);
3672  res.startNodes(&nodeId, 1);
3673  NdbSleep_SecSleep(3);
3674  ndbout_c("%u : waiting for %u to not get not-started", __LINE__, nodeId);
3675  res.waitNodesNoStart(&nodeId, 1);
3676 
3677  if (res.dumpStateOneNode(nodeId, val2, 2))
3678  return NDBT_FAILED;
3679 
3680  res.insertErrorInNode(nodeId, 9994);
3681  res.startNodes(&nodeId, 1);
3682  NdbSleep_SecSleep(3);
3683  ndbout_c("%u : waiting for %u to not get not-started", __LINE__, nodeId);
3684  res.waitNodesNoStart(&nodeId, 1);
3685 
3686  NdbSleep_SecSleep(20); // Hardcoded in ndb_mgmd (alloc timeout)
3687 
3688  ndbout_c("%u : starting %u", __LINE__, nodeId);
3689  res.startNodes(&nodeId, 1);
3690 
3691  ndbout_c("%u : waiting for cluster started", __LINE__);
3692  if (res.waitClusterStarted())
3693  {
3694  return NDBT_FAILED;
3695  }
3696  }
3697 
3698  ctx->stopTest();
3699  return NDBT_OK;
3700 }
3701 
3702 int
3703 runBug43888(NDBT_Context* ctx, NDBT_Step* step)
3704 {
3705  NdbRestarter res;
3706 
3707  if (res.getNumDbNodes() < 2)
3708  {
3709  ctx->stopTest();
3710  return NDBT_OK;
3711  }
3712 
3713  int loops = ctx->getNumLoops();
3714  while (--loops >= 0)
3715  {
3716  int master = res.getMasterNodeId();
3717  ndbout_c("master: %u", master);
3718  int nodeId = master;
3719  do {
3720  nodeId = res.getNode(NdbRestarter::NS_RANDOM);
3721  } while (nodeId == master);
3722 
3723  ndbout_c("target: %u", nodeId);
3724 
3725  res.restartOneDbNode(nodeId, false, true, true);
3729 
3730  res.waitNodesNoStart(&nodeId, 1);
3731 
3732  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3733  if (res.dumpStateOneNode(nodeId, val2, 2))
3734  return NDBT_FAILED;
3735 
3736  res.insertErrorInNode(master, 7217);
3737  res.startNodes(&nodeId, 1);
3738  NdbSleep_SecSleep(3);
3739  ndbout_c("%u : waiting for %u to not get not-started", __LINE__, nodeId);
3740  res.waitNodesNoStart(&nodeId, 1);
3741 
3742  ndbout_c("%u : starting %u", __LINE__, nodeId);
3743  res.startNodes(&nodeId, 1);
3744 
3745  ndbout_c("%u : waiting for cluster started", __LINE__);
3746  if (res.waitClusterStarted())
3747  {
3748  return NDBT_FAILED;
3749  }
3750  }
3751 
3752  ctx->stopTest();
3753  return NDBT_OK;
3754 }
3755 
3756 #define CHECK(b, m) { int _xx = b; if (!(_xx)) { \
3757  ndbout << "ERR: "<< m \
3758  << " " << "File: " << __FILE__ \
3759  << " (Line: " << __LINE__ << ")" << "- " << _xx << endl; \
3760  return NDBT_FAILED; } }
3761 
3762 int
3763 runBug44952(NDBT_Context* ctx, NDBT_Step* step)
3764 {
3765  NdbRestarter res;
3766  NdbDictionary::Dictionary* pDict = GETNDB(step)->getDictionary();
3767 
3768  const int codes [] = {
3769  5051, 5052, 5053, 0
3770  }; (void)codes;
3771 
3772  //int randomId = myRandom48(res.getNumDbNodes());
3773  //int nodeId = res.getDbNodeId(randomId);
3774 
3775  int loops = ctx->getNumLoops();
3776  const int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 } ;
3777  for (int l = 0; l < loops; l++)
3778  {
3779  int randomId = myRandom48(res.getNumDbNodes());
3780  int nodeId = res.getDbNodeId(randomId);
3781 
3782  ndbout_c("killing node %u error 5051 loop %u/%u", nodeId, l+1, loops);
3783  CHECK(res.dumpStateOneNode(nodeId, val, 2) == 0,
3784  "failed to set RestartOnErrorInsert");
3785 
3786  CHECK(res.insertErrorInNode(nodeId, 5051) == 0,
3787  "failed to insert error 5051");
3788 
3789  while (res.waitNodesNoStart(&nodeId, 1, 1 /* seconds */) != 0)
3790  {
3791  pDict->forceGCPWait();
3792  }
3793 
3794  ndbout_c("killing node %u during restart error 5052", nodeId);
3795  for (int j = 0; j < 3; j++)
3796  {
3797  ndbout_c("loop: %d - killing node %u during restart error 5052",
3798  j, nodeId);
3799  int val[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 } ;
3800  CHECK(res.dumpStateOneNode(nodeId, val, 2) == 0,
3801  "failed to set RestartOnErrorInsert");
3802 
3803  CHECK(res.insertErrorInNode(nodeId, 5052) == 0,
3804  "failed to set error insert");
3805 
3806  NdbSleep_SecSleep(3); // ...
3807 
3808  CHECK(res.startNodes(&nodeId, 1) == 0,
3809  "failed to start node");
3810 
3811  NdbSleep_SecSleep(3);
3812 
3813  CHECK(res.waitNodesNoStart(&nodeId, 1) == 0,
3814  "waitNodesNoStart failed");
3815  }
3816 
3817  CHECK(res.startNodes(&nodeId, 1) == 0,
3818  "failed to start node");
3819 
3820  CHECK(res.waitNodesStarted(&nodeId, 1) == 0,
3821  "waitNodesStarted failed");
3822  }
3823 
3824  ctx->stopTest();
3825  return NDBT_OK;
3826 }
3827 
3828 static BaseString tab_48474;
3829 
3830 int
3831 initBug48474(NDBT_Context* ctx, NDBT_Step* step)
3832 {
3833  NdbDictionary::Table tab = * ctx->getTab();
3834  NdbDictionary::Dictionary* pDict = GETNDB(step)->getDictionary();
3835 
3836  const NdbDictionary::Table * pTab = pDict->getTable(tab.getName());
3837  if (pTab == 0)
3838  return NDBT_FAILED;
3839 
3843  Uint32 cnt = 0;
3844  Vector<BaseString> tables;
3845  do
3846  {
3847  BaseString tmp;
3848  tmp.assfmt("%s_%u", tab.getName(), cnt);
3849  tab.setName(tmp.c_str());
3850 
3851  pDict->dropTable(tab.getName());
3852  if (pDict->createTable(tab) != 0)
3853  return NDBT_FAILED;
3854 
3855  const NdbDictionary::Table * pTab2 = pDict->getTable(tab.getName());
3856  if (pTab2->getObjectId() < pTab->getObjectId())
3857  {
3858  tables.push_back(tmp);
3859  }
3860  else
3861  {
3862  tab_48474 = tmp;
3863  HugoTransactions hugoTrans(* pTab2);
3864  if (hugoTrans.loadTable(GETNDB(step), 1000) != 0)
3865  {
3866  return NDBT_FAILED;
3867  }
3868  break;
3869  }
3870  cnt++;
3871  } while(true);
3872 
3873  // Now delete the extra one...
3874  for (Uint32 i = 0; i<tables.size(); i++)
3875  {
3876  pDict->dropTable(tables[i].c_str());
3877  }
3878 
3879  tables.clear();
3880 
3881  return NDBT_OK;
3882 }
3883 
3884 int
3885 runBug48474(NDBT_Context* ctx, NDBT_Step* step)
3886 {
3887  NdbRestarter res;
3888  NdbDictionary::Dictionary* pDict = GETNDB(step)->getDictionary();
3889  const NdbDictionary::Table * pTab = pDict->getTable(tab_48474.c_str());
3890  Ndb* pNdb = GETNDB(step);
3891  HugoOperations hugoOps(* pTab);
3892 
3893  int nodeId = res.getNode(NdbRestarter::NS_RANDOM);
3894  ndbout_c("restarting %d", nodeId);
3895  res.restartOneDbNode(nodeId, false, true, true);
3896  res.waitNodesNoStart(&nodeId, 1);
3897 
3898  int minlcp[] = { 7017, 1 };
3899  res.dumpStateAllNodes(minlcp, 1); // Set min time between LCP
3900 
3901  ndbout_c("starting %d", nodeId);
3902  res.startNodes(&nodeId, 1);
3903 
3904  Uint32 minutes = 5;
3905  ndbout_c("starting uncommitted transaction %u minutes", minutes);
3906  for (Uint32 m = 0; m < minutes; m++)
3907  {
3908  if (hugoOps.startTransaction(pNdb) != 0)
3909  return NDBT_FAILED;
3910 
3911  if (hugoOps.pkUpdateRecord(pNdb, 0, 50, rand()) != 0)
3912  return NDBT_FAILED;
3913 
3914  if (hugoOps.execute_NoCommit(pNdb) != 0)
3915  return NDBT_FAILED;
3916 
3917 
3918  ndbout_c("sleeping 60s");
3919  for (Uint32 i = 0; i<600 && !ctx->isTestStopped(); i++)
3920  {
3921  hugoOps.getTransaction()->refresh();
3922  NdbSleep_MilliSleep(100);
3923  }
3924 
3925  if (hugoOps.execute_Commit(pNdb) != 0)
3926  return NDBT_FAILED;
3927 
3928  hugoOps.closeTransaction(pNdb);
3929 
3930  if (ctx->isTestStopped())
3931  break;
3932  }
3933 
3934 
3935  res.dumpStateAllNodes(minlcp, 2); // reset min time between LCP
3936 
3937  ctx->stopTest();
3938  return NDBT_OK;
3939 }
3940 
3941 int
3942 cleanupBug48474(NDBT_Context* ctx, NDBT_Step* step)
3943 {
3944  NdbDictionary::Dictionary* pDict = GETNDB(step)->getDictionary();
3945  pDict->dropTable(tab_48474.c_str());
3946  return NDBT_OK;
3947 }
3948 
3949 int
3950 runBug56044(NDBT_Context* ctx, NDBT_Step* step)
3951 {
3952  int loops = ctx->getNumLoops();
3953  NdbRestarter res;
3954 
3955  if (res.getNumDbNodes() < 2)
3956  return NDBT_OK;
3957 
3958  for (int i = 0; i<loops; i++)
3959  {
3960  int master = res.getMasterNodeId();
3961  int next = res.getNextMasterNodeId(master);
3962  ndbout_c("master: %u next: %u", master, next);
3963 
3964  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
3965 
3966  if (res.dumpStateOneNode(master, val2, 2))
3967  return NDBT_FAILED;
3968 
3969  if (res.insertErrorInNode(next, 7224))
3970  return NDBT_FAILED;
3971 
3972  res.waitNodesNoStart(&master, 1);
3973  res.startNodes(&master, 1);
3974  if (res.waitClusterStarted() != 0)
3975  return NDBT_FAILED;
3976  }
3977 
3978  return NDBT_OK;
3979 }
3980 
3981 int
3982 runBug57767(NDBT_Context* ctx, NDBT_Step* step)
3983 {
3984  NdbRestarter res;
3985 
3986  if (res.getNumDbNodes() < 2)
3987  return NDBT_OK;
3988 
3989  int node0 = res.getNode(NdbRestarter::NS_RANDOM);
3990  int node1 = res.getRandomNodeSameNodeGroup(node0, rand());
3991  ndbout_c("%u %u", node0, node1);
3992 
3993  res.restartOneDbNode(node0, false, true, true);
3994  res.waitNodesNoStart(&node0, 1);
3995  res.insertErrorInNode(node0, 1000);
3996 
3997  HugoTransactions hugoTrans(*ctx->getTab());
3998  hugoTrans.scanUpdateRecords(GETNDB(step), 0);
3999 
4000  res.insertErrorInNode(node1, 5060);
4001  res.startNodes(&node0, 1);
4002  res.waitClusterStarted();
4003  return NDBT_OK;
4004 }
4005 
4006 int
4007 runBug57522(NDBT_Context* ctx, NDBT_Step* step)
4008 {
4009  int loops = ctx->getNumLoops();
4010  NdbRestarter res;
4011 
4012  if (res.getNumDbNodes() < 4)
4013  return NDBT_OK;
4014 
4015  for (int i = 0; i<loops; i++)
4016  {
4017  int master = res.getMasterNodeId();
4018  int next0 = res.getNextMasterNodeId(master);
4019  int next1 = res.getNextMasterNodeId(next0);
4020  ndbout_c("master: %d next0: %d next1: %d", master, next0, next1);
4021 
4022  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
4023 
4024  if (res.dumpStateOneNode(master, val2, 2))
4025  return NDBT_FAILED;
4026 
4027  int val3[] = { 7999, 7226, next1 };
4028  if (res.dumpStateOneNode(master, val3, 3))
4029  return NDBT_FAILED;
4030 
4031  res.waitNodesNoStart(&master, 1);
4032  res.startNodes(&master, 1);
4033  if (res.waitClusterStarted() != 0)
4034  return NDBT_FAILED;
4035  }
4036 
4037  return NDBT_OK;
4038 }
4039 
4040 int
4041 runForceStopAndRestart(NDBT_Context* ctx, NDBT_Step* step)
4042 {
4043  NdbRestarter res;
4044  if (res.getNumDbNodes() < 2)
4045  return NDBT_OK;
4046 
4047  Vector<int> group1;
4048  Vector<int> group2;
4049  Bitmask<256/32> nodeGroupMap;
4050  for (int j = 0; j<res.getNumDbNodes(); j++)
4051  {
4052  int node = res.getDbNodeId(j);
4053  int ng = res.getNodeGroup(node);
4054  if (nodeGroupMap.get(ng))
4055  {
4056  group2.push_back(node);
4057  }
4058  else
4059  {
4060  group1.push_back(node);
4061  nodeGroupMap.set(ng);
4062  }
4063  }
4064 
4065  printf("group1: ");
4066  for (size_t i = 0; i<group1.size(); i++)
4067  printf("%d ", group1[i]);
4068  printf("\n");
4069 
4070  printf("group2: ");
4071  for (size_t i = 0; i<group2.size(); i++)
4072  printf("%d ", group2[i]);
4073  printf("\n");
4074 
4075  // Stop half of the cluster
4076  res.restartNodes(group1.getBase(), (int)group1.size(),
4077  NdbRestarter::NRRF_NOSTART | NdbRestarter::NRRF_ABORT);
4078  res.waitNodesNoStart(group1.getBase(), (int)group1.size());
4079 
4080  ndbout_c("%u", __LINE__);
4081  // Try to stop first node in second half without force, should return error
4082  if (res.restartOneDbNode(group2[0],
4083  false, /* initial */
4084  true, /* nostart */
4085  false, /* abort */
4086  false /* force */) != -1)
4087  {
4088  ndbout_c("%u", __LINE__);
4089  g_err << "Restart suceeded without force" << endl;
4090  return NDBT_FAILED;
4091  }
4092 
4093  ndbout_c("%u", __LINE__);
4094 
4095  // Now stop with force
4096  if (res.restartOneDbNode(group2[0],
4097  false, /* initial */
4098  true, /* nostart */
4099  false, /* abort */
4100  true /* force */) != 0)
4101  {
4102  ndbout_c("%u", __LINE__);
4103  g_err << "Could not restart with force" << endl;
4104  return NDBT_FAILED;
4105  }
4106 
4107  ndbout_c("%u", __LINE__);
4108 
4109  // All nodes should now be in nostart, the above stop force
4110  // cvaused the remainig nodes to be stopped(and restarted nostart)
4111  res.waitClusterNoStart();
4112 
4113  ndbout_c("%u", __LINE__);
4114 
4115  // Start second half back up again
4116  res.startNodes(group2.getBase(), (int)group2.size());
4117  res.waitNodesStarted(group2.getBase(), (int)group2.size());
4118 
4119  ndbout_c("%u", __LINE__);
4120 
4121  // Try to stop remaining half without force, should return error
4122  if (res.restartNodes(group2.getBase(), (int)group2.size(),
4123  NdbRestarter::NRRF_NOSTART) != -1)
4124  {
4125  g_err << "Restart suceeded without force" << endl;
4126  return NDBT_FAILED;
4127  }
4128 
4129  ndbout_c("%u", __LINE__);
4130 
4131  // Now stop with force
4132  if (res.restartNodes(group2.getBase(), (int)group2.size(),
4133  NdbRestarter::NRRF_NOSTART |
4134  NdbRestarter::NRRF_FORCE) != 0)
4135  {
4136  g_err << "Could not restart with force" << endl;
4137  return NDBT_FAILED;
4138  }
4139 
4140  ndbout_c("%u", __LINE__);
4141 
4142  if (res.waitNodesNoStart(group2.getBase(), (int)group2.size()))
4143  {
4144  g_err << "Failed to waitNodesNoStart" << endl;
4145  return NDBT_FAILED;
4146  }
4147 
4148  // Start all nodes again
4149  res.startAll();
4150  res.waitClusterStarted();
4151 
4152  return NDBT_OK;
4153 }
4154 
4155 int
4156 runBug58453(NDBT_Context* ctx, NDBT_Step* step)
4157 {
4158  NdbRestarter res;
4159  if (res.getNumDbNodes() < 4)
4160  return NDBT_OK;
4161 
4162  Ndb* pNdb = GETNDB(step);
4163  HugoOperations hugoOps(*ctx->getTab());
4164 
4165  int loops = ctx->getNumLoops();
4166  while (loops--)
4167  {
4168  if (hugoOps.startTransaction(pNdb) != 0)
4169  return NDBT_FAILED;
4170 
4171  if (hugoOps.pkInsertRecord(pNdb, 0, 128 /* records */) != 0)
4172  return NDBT_FAILED;
4173 
4174  int err = 5062;
4175  switch(loops & 1){
4176  case 0:
4177  err = 5062;
4178  break;
4179  case 1:
4180  err = 5063;
4181  break;
4182  }
4183  int node = (int)hugoOps.getTransaction()->getConnectedNodeId();
4184  int node0 = res.getRandomNodeOtherNodeGroup(node, rand());
4185  int node1 = res.getRandomNodeSameNodeGroup(node0, rand());
4186 
4187  ndbout_c("node %u err: %u, node: %u err: %u",
4188  node0, 5061, node1, err);
4189 
4190  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
4191 
4192  res.dumpStateOneNode(node, val2, 2);
4193  res.insertErrorInNode(node0, 5061);
4194  res.insertErrorInNode(node1, err);
4195 
4196  hugoOps.execute_Commit(pNdb);
4197  hugoOps.closeTransaction(pNdb);
4198 
4199  res.waitNodesNoStart(&node, 1);
4200  res.startNodes(&node, 1);
4201  res.waitClusterStarted();
4202  hugoOps.clearTable(pNdb);
4203  }
4204 
4205  return NDBT_OK;
4206 }
4207 
4208 
4209 
4210 int runRestartToDynamicOrder(NDBT_Context* ctx, NDBT_Step* step)
4211 {
4212  /* Here we perform node restarts to get the various node's
4213  * dynamic ids in a particular order
4214  * This affects which nodes heartbeat which (low -> high)
4215  * and which is the president (lowest).
4216  * Each restarting node gets a higher dynamic id, so the
4217  * first node restarted will eventually become president
4218  * Note that we're assuming NoOfReplicas == 2 here.
4219  */
4220  /* TODO :
4221  * Refactor into
4222  * 1) Get current cluster dynorder info
4223  * 2) Choose a dynorder info
4224  * 3) Restart to given dynorder if necessary
4225  */
4226  Uint32 dynOrder = ctx->getProperty("DynamicOrder", Uint32(0));
4227  NdbRestarter restarter;
4228  Uint32 numNodes = restarter.getNumDbNodes();
4229 
4230  Vector<Uint32> currOrder;
4231  Vector<Uint32> newOrder;
4232  Vector<Uint32> odds;
4233  Vector<Uint32> evens;
4234 
4235  if (numNodes == 2)
4236  {
4237  ndbout_c("No Dynamic reordering possible with 2 nodes");
4238  return NDBT_OK;
4239  }
4240  if (numNodes & 1)
4241  {
4242  ndbout_c("Non multiple-of-2 number of nodes. Not supported");
4243  return NDBT_FAILED;
4244  }
4245 
4246  Uint32 master = restarter.getMasterNodeId();
4247 
4248  for (Uint32 n=0; n < numNodes; n++)
4249  {
4250  currOrder.push_back(master);
4251  master = restarter.getNextMasterNodeId(master);
4252  }
4253 
4254  for (Uint32 n=0; n < numNodes; n++)
4255  {
4256  Uint32 nodeId = restarter.getDbNodeId(n);
4257  if (nodeId & 1)
4258  {
4259  odds.push_back(nodeId);
4260  }
4261  else
4262  {
4263  evens.push_back(nodeId);
4264  }
4265  }
4266 
4267  if (odds.size() != evens.size())
4268  {
4269  ndbout_c("Failed - odds.size() (%u) != evens.size() (%u)",
4270  odds.size(),
4271  evens.size());
4272  return NDBT_FAILED;
4273  }
4274 
4275  ndbout_c("Current dynamic ordering : ");
4276  for (Uint32 n=0; n<numNodes; n++)
4277  {
4278  ndbout_c(" %u %s", currOrder[n], ((n==0)?"*":""));
4279  }
4280 
4281  if (dynOrder == 0)
4282  {
4283  ndbout_c("No change in dynamic order");
4284  return NDBT_OK;
4285  }
4286 
4287  Uint32 control= dynOrder - 1;
4288 
4289  bool oddPresident = control & 1;
4290  bool interleave = control & 2;
4291  bool reverseSideA = control & 4;
4292  bool reverseSideB = control & 8;
4293 
4294  /* Odds first Interleave O/E Reverse A Reverse B
4295  * 1 N N N N
4296  * 2 Y N N N
4297  * 3 N Y N N
4298  * 4 Y Y N N
4299  * 5 N N Y N
4300  * 6 Y N Y N
4301  * 7 N Y Y N
4302  * 8 Y Y Y N
4303  * 9 N N N Y
4304  * 10 Y N N Y
4305  * 11 N Y N Y
4306  * 12 Y Y N Y
4307  * 13 N N Y Y
4308  * 14 Y N Y Y
4309  * 15 N Y Y Y
4310  * 16 Y Y Y Y
4311  *
4312  * Interesting values
4313  * 1) Even first, no interleave, no reverse
4314  * e.g. 2->4->6->3->5->7
4315  * 2) Odd first, no interleave, no reverse
4316  * e.g. 3->5->7->2->4->6
4317  * 3) Even first, interleave, no reverse
4318  * e.g. 2->3->4->5->6->7
4319  * 9) Even first, no interleave, reverse B
4320  * e.g. 2->4->6->7->5->3
4321  *
4322  * 'First' node becomes president.
4323  * Which node(s) monitor president affects when
4324  * arbitration may be required
4325  */
4326 
4327  ndbout_c("Generating ordering with %s president, sides %sinterleaved",
4328  (oddPresident?"odd": "even"),
4329  (interleave?"":"not "));
4330  if (reverseSideA)
4331  ndbout_c(" %s reversed", (oddPresident?"odds": "evens"));
4332 
4333  if (reverseSideB)
4334  ndbout_c(" %s reversed", (oddPresident?"evens": "odds"));
4335 
4336  Vector<Uint32>* sideA;
4337  Vector<Uint32>* sideB;
4338 
4339  if (oddPresident)
4340  {
4341  sideA = &odds;
4342  sideB = &evens;
4343  }
4344  else
4345  {
4346  sideA = &evens;
4347  sideB = &odds;
4348  }
4349 
4350  if (interleave)
4351  {
4352  for (Uint32 n=0; n < sideA->size(); n++)
4353  {
4354  Uint32 indexA = reverseSideA? (sideA->size() - (n+1)) : n;
4355  newOrder.push_back((*sideA)[indexA]);
4356  Uint32 indexB = reverseSideB? (sideB->size() - (n+1)) : n;
4357  newOrder.push_back((*sideB)[indexB]);
4358  }
4359  }
4360  else
4361  {
4362  for (Uint32 n=0; n < sideA->size(); n++)
4363  {
4364  Uint32 indexA = reverseSideA? (sideA->size() - (n+1)) : n;
4365  newOrder.push_back((*sideA)[indexA]);
4366  }
4367  for (Uint32 n=0; n < sideB->size(); n++)
4368  {
4369  Uint32 indexB = reverseSideB? (sideB->size() - (n+1)) : n;
4370  newOrder.push_back((*sideB)[indexB]);
4371  }
4372  }
4373 
4374 
4375  bool diff = false;
4376  for (Uint32 n=0; n < newOrder.size(); n++)
4377  {
4378  ndbout_c(" %u %s",
4379  newOrder[n],
4380  ((n==0)?"*":" "));
4381 
4382  diff |= (newOrder[n] != currOrder[n]);
4383  }
4384 
4385  if (!diff)
4386  {
4387  ndbout_c("Cluster already in correct configuration");
4388  return NDBT_OK;
4389  }
4390 
4391  for (Uint32 n=0; n < newOrder.size(); n++)
4392  {
4393  ndbout_c("Now restarting node %u", newOrder[n]);
4394  if (restarter.restartOneDbNode(newOrder[n],
4395  false, // initial
4396  true, // nostart
4397  true) // abort
4398  != NDBT_OK)
4399  {
4400  ndbout_c("Failed to restart node");
4401  return NDBT_FAILED;
4402  }
4403  if (restarter.waitNodesNoStart((const int*) &newOrder[n], 1) != NDBT_OK)
4404  {
4405  ndbout_c("Failed waiting for node to enter NOSTART state");
4406  return NDBT_FAILED;
4407  }
4408  if (restarter.startNodes((const int*) &newOrder[n], 1) != NDBT_OK)
4409  {
4410  ndbout_c("Failed to start node");
4411  return NDBT_FAILED;
4412  }
4413  if (restarter.waitNodesStarted((const int*) &newOrder[n], 1) != NDBT_OK)
4414  {
4415  ndbout_c("Failed waiting for node to start");
4416  return NDBT_FAILED;
4417  }
4418  ndbout_c(" Done.");
4419  }
4420 
4421  ndbout_c("All restarts completed. NdbRestarter says master is %u",
4422  restarter.getMasterNodeId());
4423  if (restarter.getMasterNodeId() != (int) newOrder[0])
4424  {
4425  ndbout_c(" Should be %u, failing", newOrder[0]);
4426  return NDBT_FAILED;
4427  }
4428 
4429  return NDBT_OK;
4430 }
4431 
4432 struct NodeGroupMembers
4433 {
4434  Uint32 ngid;
4435  Uint32 membCount;
4436  Uint32 members[4];
4437 };
4438 
4439 template class Vector<NodeGroupMembers>;
4440 
4441 int analyseDynamicOrder(NDBT_Context* ctx, NDBT_Step* step)
4442 {
4443  NdbRestarter restarter;
4444  Uint32 numNodes = restarter.getNumDbNodes();
4445  Uint32 master = restarter.getMasterNodeId();
4446  Vector<Uint32> dynamicOrder;
4447  Vector<Uint32> nodeGroup;
4448  Vector<Uint32> monitorsNode;
4449  Vector<Uint32> monitoredByNode;
4450  Vector<Uint32> monitorsRemote;
4451  Vector<Uint32> remoteMonitored;
4452  Vector<Uint32> sameNGMonitored;
4453  Vector<Uint32> distanceToRemote;
4454  Vector<Uint32> nodeIdToDynamicIndex;
4455  Uint32 maxDistanceToRemoteLink = 0;
4456 
4457  /* TODO :
4458  * Refactor into :
4459  * 1) Determine dynorder from running cluster
4460  * 2) Analyse dynorder in general
4461  * 3) Analyse dynorder from point of view of latency split
4462  *
4463  * 4) Support splits other than odd/even total
4464  * - Partial split
4465  * - Some link failures
4466  */
4467 
4468  /* Determine dynamic order from running cluster */
4469  for (Uint32 n=0; n < numNodes; n++)
4470  {
4471  dynamicOrder.push_back(master);
4472  nodeGroup.push_back(restarter.getNodeGroup(master));
4473  master = restarter.getNextMasterNodeId(master);
4474  Uint32 zero=0;
4475  nodeIdToDynamicIndex.set(n, master, zero);
4476  }
4477 
4478  /* Look at implied HB links */
4479  for (Uint32 n=0; n < numNodes; n++)
4480  {
4481  Uint32 nodeId = dynamicOrder[n];
4482  Uint32 monitoredByIndex = (n+1) % numNodes;
4483  Uint32 monitorsIndex = (n+ numNodes - 1) % numNodes;
4484  monitoredByNode.push_back(dynamicOrder[monitoredByIndex]);
4485  monitorsNode.push_back(dynamicOrder[monitorsIndex]);
4486  remoteMonitored.push_back((nodeId & 1) != (monitoredByNode[n] & 1));
4487  monitorsRemote.push_back((nodeId & 1) != (monitorsNode[n] & 1));
4488  sameNGMonitored.push_back(nodeGroup[n] == nodeGroup[monitoredByIndex]);
4489  }
4490 
4491  /* Look at split implications */
4492  for (Uint32 n=0; n < numNodes; n++)
4493  {
4494  Uint32 distanceToRemoteHBLink = 0;
4495  for (Uint32 m=0; m < numNodes; m++)
4496  {
4497  if (remoteMonitored[n+m])
4498  break;
4499  distanceToRemoteHBLink++;
4500  }
4501 
4502  distanceToRemote.push_back(distanceToRemoteHBLink);
4503  maxDistanceToRemoteLink = MAX(maxDistanceToRemoteLink, distanceToRemoteHBLink);
4504  }
4505 
4506 
4507  ndbout_c("Dynamic order analysis");
4508 
4509  for (Uint32 n=0; n < numNodes; n++)
4510  {
4511  ndbout_c(" %u %s %u%s%u%s%u \t Monitored by %s nodegroup, Dist to remote link : %u",
4512  dynamicOrder[n],
4513  ((n==0)?"*":" "),
4514  monitorsNode[n],
4515  ((monitorsRemote[n])?" >":"-->"),
4516  dynamicOrder[n],
4517  ((remoteMonitored[n])?" >":"-->"),
4518  monitoredByNode[n],
4519  ((sameNGMonitored[n])?"same":"other"),
4520  distanceToRemote[n]);
4521  }
4522 
4523  ndbout_c("\n");
4524 
4525  Vector<NodeGroupMembers> nodeGroupMembers;
4526 
4527  for (Uint32 n=0; n < numNodes; n++)
4528  {
4529  Uint32 ng = nodeGroup[n];
4530 
4531  bool ngfound = false;
4532  for (Uint32 m = 0; m < nodeGroupMembers.size(); m++)
4533  {
4534  if (nodeGroupMembers[m].ngid == ng)
4535  {
4536  NodeGroupMembers& ngInfo = nodeGroupMembers[m];
4537  ngInfo.members[ngInfo.membCount++] = dynamicOrder[n];
4538  ngfound = true;
4539  break;
4540  }
4541  }
4542 
4543  if (!ngfound)
4544  {
4545  NodeGroupMembers newGroupInfo;
4546  newGroupInfo.ngid = ng;
4547  newGroupInfo.membCount = 1;
4548  newGroupInfo.members[0] = dynamicOrder[n];
4549  nodeGroupMembers.push_back(newGroupInfo);
4550  }
4551  }
4552 
4553  ndbout_c("Nodegroups");
4554 
4555  for (Uint32 n=0; n < nodeGroupMembers.size(); n++)
4556  {
4557  ndbout << " " << nodeGroupMembers[n].ngid << " (";
4558  bool allRemoteMonitored = true;
4559  for (Uint32 m=0; m < nodeGroupMembers[n].membCount; m++)
4560  {
4561  Uint32 nodeId = nodeGroupMembers[n].members[m];
4562  ndbout << nodeId;
4563  if ((m+1) < nodeGroupMembers[n].membCount)
4564  ndbout << ",";
4565  Uint32 dynamicIndex = nodeIdToDynamicIndex[nodeId];
4566  allRemoteMonitored &= remoteMonitored[dynamicIndex];
4567  }
4568  ndbout << ") Entirely remote monitored NGs risk : "
4569  << (allRemoteMonitored?"Y":"N") << "\n";
4570  }
4571  ndbout_c("\n");
4572 
4573  ndbout_c("Cluster-split latency behaviour");
4574 
4575  Uint32 oddPresident = dynamicOrder[0];
4576  Uint32 evenPresident = dynamicOrder[0];
4577 
4578  for (Uint32 n=0; n <= maxDistanceToRemoteLink; n++)
4579  {
4580  Vector<Uint32> failedNodeGroups;
4581  ndbout << " " << n <<" HB latency period(s), nodes (";
4582  bool useComma = false;
4583  bool presidentFailed = false;
4584  for (Uint32 m=0; m < numNodes; m++)
4585  {
4586  if (distanceToRemote[m] == n)
4587  {
4588  Uint32 failingNodeId = dynamicOrder[m];
4589  if (useComma)
4590  ndbout << ",";
4591 
4592  useComma = true;
4593  ndbout << failingNodeId;
4594 
4595  if ((failingNodeId == evenPresident) ||
4596  (failingNodeId == oddPresident))
4597  {
4598  ndbout << "*";
4599  presidentFailed = true;
4600  }
4601 
4602  {
4603  Uint32 ng = nodeGroup[m];
4604  for (Uint32 i=0; i< nodeGroupMembers.size(); i++)
4605  {
4606  if (nodeGroupMembers[i].ngid == ng)
4607  {
4608  if ((--nodeGroupMembers[i].membCount) == 0)
4609  {
4610  failedNodeGroups.push_back(ng);
4611  }
4612  }
4613  }
4614  }
4615  }
4616  }
4617  ndbout << ") will be declared failed." << endl;
4618  if (failedNodeGroups.size() != 0)
4619  {
4620  ndbout << " NG failure risk on reconnect for nodegroups : ";
4621  for (Uint32 i=0; i< failedNodeGroups.size(); i++)
4622  {
4623  if (i > 0)
4624  ndbout << ",";
4625  ndbout << failedNodeGroups[i];
4626  }
4627  ndbout << endl;
4628  }
4629  if (presidentFailed)
4630  {
4631  /* A president (even/odd/both) has failed, we should
4632  * calculate the new president(s) from the p.o.v.
4633  * of both sides
4634  */
4635  Uint32 newOdd=0;
4636  Uint32 newEven=0;
4637  for (Uint32 i=0; i< numNodes; i++)
4638  {
4639  /* Each side finds either the first node on their
4640  * side, or the first node on the other side which
4641  * is still 'alive' from their point of view
4642  */
4643  bool candidateIsOdd = dynamicOrder[i] & 1;
4644 
4645  if (!newOdd)
4646  {
4647  if (candidateIsOdd ||
4648  (distanceToRemote[i] > n))
4649  {
4650  newOdd = dynamicOrder[i];
4651  }
4652  }
4653  if (!newEven)
4654  {
4655  if ((!candidateIsOdd) ||
4656  (distanceToRemote[i] > n))
4657  {
4658  newEven = dynamicOrder[i];
4659  }
4660  }
4661  }
4662 
4663  bool oddPresidentFailed = (oddPresident != newOdd);
4664  bool evenPresidentFailed = (evenPresident != newEven);
4665 
4666  if (oddPresidentFailed)
4667  {
4668  ndbout_c(" Odd president (%u) failed, new odd president : %u",
4669  oddPresident, newOdd);
4670  oddPresident = newOdd;
4671  }
4672  if (evenPresidentFailed)
4673  {
4674  ndbout_c(" Even president (%u) failed, new even president : %u",
4675  evenPresident, newEven);
4676  evenPresident = newEven;
4677  }
4678 
4679  if (oddPresident != evenPresident)
4680  {
4681  ndbout_c(" President role duplicated, Odd (%u), Even (%u)",
4682  oddPresident, evenPresident);
4683  }
4684 
4685  }
4686  }
4687 
4688  ndbout << endl << endl;
4689 
4690  return NDBT_OK;
4691 }
4692 
4693 int runSplitLatency25PctFail(NDBT_Context* ctx, NDBT_Step* step)
4694 {
4695  /* Use dump commands to inject artificial inter-node latency
4696  * Use an error insert to cause latency to disappear when
4697  * a node observes > 25% of nodes failed.
4698  * This should trigger a race of FAIL_REQs from both sides
4699  * of the cluster, and can result in cluster failure
4700  */
4701  NdbRestarter restarter;
4702 
4703  /*
4704  * First set the ConnectCheckIntervalDelay to 1500
4705  */
4706  {
4707  int dump[] = { 9994, 1500 };
4708  restarter.dumpStateAllNodes(dump, 2);
4709  }
4710 
4711  {
4712  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
4713  restarter.dumpStateAllNodes(val2, 2);
4714  }
4715 
4716  /* First the error insert which will drop latency (QMGR) */
4717  restarter.insertErrorInAllNodes(938);
4718 
4719  /* Now the dump code which causes the system to experience
4720  * latency along odd/even lines (CMVMI)
4721  *
4722  */
4723  int dumpStateArgs[] = {9990, 1};
4724  restarter.dumpStateAllNodes(dumpStateArgs, 2);
4725 
4729  ndbout_c("Waiting for half of cluster to die");
4730  int not_started = 0;
4731  const int node_count = restarter.getNumDbNodes();
4732  do
4733  {
4734  not_started = 0;
4735  for (int i = 0; i < node_count; i++)
4736  {
4737  if (restarter.getNodeStatus(restarter.getDbNodeId(i)) == NDB_MGM_NODE_STATUS_NOT_STARTED)
4738  not_started++;
4739  }
4740  } while (2 * not_started != node_count);
4741 
4742  ndbout_c("Restarting cluster");
4743  restarter.restartAll(false, true, true);
4744  ndbout_c("Waiting cluster not started");
4745  restarter.waitClusterNoStart();
4746 
4747  ndbout_c("Starting");
4748  restarter.startAll();
4749  restarter.waitClusterStarted();
4750 
4751  return NDBT_OK;
4752 }
4753 
4754 NDBT_TESTSUITE(testNodeRestart);
4755 TESTCASE("NoLoad",
4756  "Test that one node at a time can be stopped and then restarted "\
4757  "when there are no load on the system. Do this loop number of times"){
4758  INITIALIZER(runCheckAllNodesStarted);
4759  INITIALIZER(runLoadTable);
4760  STEP(runRestarter);
4761  FINALIZER(runClearTable);
4762 }
4763 TESTCASE("PkRead",
4764  "Test that one node at a time can be stopped and then restarted "\
4765  "perform pk read while restarting. Do this loop number of times"){
4766  TC_PROPERTY("ReadLockMode", NdbOperation::LM_Read);
4767  INITIALIZER(runCheckAllNodesStarted);
4768  INITIALIZER(runLoadTable);
4769  STEP(runRestarter);
4770  STEP(runPkReadUntilStopped);
4771  FINALIZER(runClearTable);
4772 }
4773 TESTCASE("PkReadCommitted",
4774  "Test that one node at a time can be stopped and then restarted "\
4775  "perform pk read while restarting. Do this loop number of times"){
4776  TC_PROPERTY("ReadLockMode", NdbOperation::LM_CommittedRead);
4777  INITIALIZER(runCheckAllNodesStarted);
4778  INITIALIZER(runLoadTable);
4779  STEP(runRestarter);
4780  STEP(runPkReadUntilStopped);
4781  FINALIZER(runClearTable);
4782 }
4783 TESTCASE("MixedPkRead",
4784  "Test that one node at a time can be stopped and then restarted "\
4785  "perform pk read while restarting. Do this loop number of times"){
4786  TC_PROPERTY("ReadLockMode", Uint32(-1));
4787  INITIALIZER(runCheckAllNodesStarted);
4788  INITIALIZER(runLoadTable);
4789  STEP(runRestarter);
4790  STEP(runPkReadUntilStopped);
4791  FINALIZER(runClearTable);
4792 }
4793 TESTCASE("PkReadPkUpdate",
4794  "Test that one node at a time can be stopped and then restarted "\
4795  "perform pk read and pk update while restarting. Do this loop number of times"){
4796  TC_PROPERTY("ReadLockMode", NdbOperation::LM_Read);
4797  INITIALIZER(runCheckAllNodesStarted);
4798  INITIALIZER(runLoadTable);
4799  STEP(runRestarter);
4800  STEP(runPkReadUntilStopped);
4801  STEP(runPkUpdateUntilStopped);
4802  STEP(runPkReadPkUpdateUntilStopped);
4803  STEP(runPkReadUntilStopped);
4804  STEP(runPkUpdateUntilStopped);
4805  STEP(runPkReadPkUpdateUntilStopped);
4806  FINALIZER(runClearTable);
4807 }
4808 TESTCASE("MixedPkReadPkUpdate",
4809  "Test that one node at a time can be stopped and then restarted "\
4810  "perform pk read and pk update while restarting. Do this loop number of times"){
4811  TC_PROPERTY("ReadLockMode", Uint32(-1));
4812  INITIALIZER(runCheckAllNodesStarted);
4813  INITIALIZER(runLoadTable);
4814  STEP(runRestarter);
4815  STEP(runPkReadUntilStopped);
4816  STEP(runPkUpdateUntilStopped);
4817  STEP(runPkReadPkUpdateUntilStopped);
4818  STEP(runPkReadUntilStopped);
4819  STEP(runPkUpdateUntilStopped);
4820  STEP(runPkReadPkUpdateUntilStopped);
4821  FINALIZER(runClearTable);
4822 }
4823 TESTCASE("ReadUpdateScan",
4824  "Test that one node at a time can be stopped and then restarted "\
4825  "perform pk read, pk update and scan reads while restarting. Do this loop number of times"){
4826  INITIALIZER(runCheckAllNodesStarted);
4827  INITIALIZER(runLoadTable);
4828  STEP(runRestarter);
4829  STEP(runPkReadUntilStopped);
4830  STEP(runPkUpdateUntilStopped);
4831  STEP(runPkReadPkUpdateUntilStopped);
4832  STEP(runScanReadUntilStopped);
4833  STEP(runScanUpdateUntilStopped);
4834  FINALIZER(runClearTable);
4835 }
4836 TESTCASE("MixedReadUpdateScan",
4837  "Test that one node at a time can be stopped and then restarted "\
4838  "perform pk read, pk update and scan reads while restarting. Do this loop number of times"){
4839  TC_PROPERTY("ReadLockMode", Uint32(-1));
4840  INITIALIZER(runCheckAllNodesStarted);
4841  INITIALIZER(runLoadTable);
4842  STEP(runRestarter);
4843  STEP(runPkReadUntilStopped);
4844  STEP(runPkUpdateUntilStopped);
4845  STEP(runPkReadPkUpdateUntilStopped);
4846  STEP(runScanReadUntilStopped);
4847  STEP(runScanUpdateUntilStopped);
4848  FINALIZER(runClearTable);
4849 }
4850 TESTCASE("Terror",
4851  "Test that one node at a time can be stopped and then restarted "\
4852  "perform all kind of transactions while restarting. Do this loop number of times"){
4853  INITIALIZER(runCheckAllNodesStarted);
4854  INITIALIZER(runLoadTable);
4855  STEP(runRestarter);
4856  STEP(runPkReadUntilStopped);
4857  STEP(runPkUpdateUntilStopped);
4858  STEP(runScanReadUntilStopped);
4859  STEP(runScanUpdateUntilStopped);
4860  FINALIZER(runClearTable);
4861 }
4862 TESTCASE("FullDb",
4863  "Test that one node at a time can be stopped and then restarted "\
4864  "when db is full. Do this loop number of times"){
4865  INITIALIZER(runCheckAllNodesStarted);
4866  INITIALIZER(runFillTable);
4867  STEP(runRestarter);
4868 }
4869 TESTCASE("RestartRandomNode",
4870  "Test that we can execute the restart RestartRandomNode loop\n"\
4871  "number of times"){
4872  INITIALIZER(runCheckAllNodesStarted);
4873  INITIALIZER(runLoadTable);
4874  STEP(runRestarts);
4875  FINALIZER(runScanReadVerify);
4876  FINALIZER(runClearTable);
4877 }
4878 TESTCASE("RestartRandomNodeError",
4879  "Test that we can execute the restart RestartRandomNodeError loop\n"\
4880  "number of times"){
4881  INITIALIZER(runCheckAllNodesStarted);
4882  INITIALIZER(runLoadTable);
4883  STEP(runRestarts);
4884  FINALIZER(runScanReadVerify);
4885  FINALIZER(runClearTable);
4886 }
4887 TESTCASE("RestartRandomNodeInitial",
4888  "Test that we can execute the restart RestartRandomNodeInitial loop\n"\
4889  "number of times"){
4890  INITIALIZER(runCheckAllNodesStarted);
4891  INITIALIZER(runLoadTable);
4892  STEP(runRestarts);
4893  FINALIZER(runScanReadVerify);
4894  FINALIZER(runClearTable);
4895 }
4896 TESTCASE("RestartNFDuringNR",
4897  "Test that we can execute the restart RestartNFDuringNR loop\n"\
4898  "number of times"){
4899  INITIALIZER(runCheckAllNodesStarted);
4900  INITIALIZER(runLoadTable);
4901  STEP(runRestarts);
4902  STEP(runPkUpdateUntilStopped);
4903  STEP(runScanUpdateUntilStopped);
4904  FINALIZER(runScanReadVerify);
4905  FINALIZER(runClearTable);
4906 }
4907 TESTCASE("RestartMasterNodeError",
4908  "Test that we can execute the restart RestartMasterNodeError loop\n"\
4909  "number of times"){
4910  INITIALIZER(runCheckAllNodesStarted);
4911  INITIALIZER(runLoadTable);
4912  STEP(runRestarts);
4913  FINALIZER(runScanReadVerify);
4914  FINALIZER(runClearTable);
4915 }
4916 
4917 TESTCASE("TwoNodeFailure",
4918  "Test that we can execute the restart TwoNodeFailure\n"\
4919  "(which is a multiple node failure restart) loop\n"\
4920  "number of times"){
4921  INITIALIZER(runCheckAllNodesStarted);
4922  INITIALIZER(runLoadTable);
4923  STEP(runRestarts);
4924  FINALIZER(runScanReadVerify);
4925  FINALIZER(runClearTable);
4926 }
4927 TESTCASE("TwoMasterNodeFailure",
4928  "Test that we can execute the restart TwoMasterNodeFailure\n"\
4929  "(which is a multiple node failure restart) loop\n"\
4930  "number of times"){
4931  INITIALIZER(runCheckAllNodesStarted);
4932  INITIALIZER(runLoadTable);
4933  STEP(runRestarts);
4934  FINALIZER(runScanReadVerify);
4935  FINALIZER(runClearTable);
4936 }
4937 TESTCASE("FiftyPercentFail",
4938  "Test that we can execute the restart FiftyPercentFail\n"\
4939  "(which is a multiple node failure restart) loop\n"\
4940  "number of times"){
4941  INITIALIZER(runCheckAllNodesStarted);
4942  INITIALIZER(runLoadTable);
4943  STEP(runRestarts);
4944  FINALIZER(runScanReadVerify);
4945  FINALIZER(runClearTable);
4946 }
4947 TESTCASE("RestartAllNodes",
4948  "Test that we can execute the restart RestartAllNodes\n"\
4949  "(which is a system restart) loop\n"\
4950  "number of times"){
4951  INITIALIZER(runCheckAllNodesStarted);
4952  INITIALIZER(runLoadTable);
4953  STEP(runRestarts);
4954  FINALIZER(runScanReadVerify);
4955  FINALIZER(runClearTable);
4956 }
4957 TESTCASE("RestartAllNodesAbort",
4958  "Test that we can execute the restart RestartAllNodesAbort\n"\
4959  "(which is a system restart) loop\n"\
4960  "number of times"){
4961  INITIALIZER(runCheckAllNodesStarted);
4962  INITIALIZER(runLoadTable);
4963  STEP(runRestarts);
4964  FINALIZER(runScanReadVerify);
4965  FINALIZER(runClearTable);
4966 }
4967 TESTCASE("RestartAllNodesError9999",
4968  "Test that we can execute the restart RestartAllNodesError9999\n"\
4969  "(which is a system restart) loop\n"\
4970  "number of times"){
4971  INITIALIZER(runCheckAllNodesStarted);
4972  INITIALIZER(runLoadTable);
4973  STEP(runRestarts);
4974  FINALIZER(runScanReadVerify);
4975  FINALIZER(runClearTable);
4976 }
4977 TESTCASE("FiftyPercentStopAndWait",
4978  "Test that we can execute the restart FiftyPercentStopAndWait\n"\
4979  "(which is a system restart) loop\n"\
4980  "number of times"){
4981  INITIALIZER(runCheckAllNodesStarted);
4982  INITIALIZER(runLoadTable);
4983  STEP(runRestarts);
4984  FINALIZER(runScanReadVerify);
4985  FINALIZER(runClearTable);
4986 }
4987 TESTCASE("RestartNodeDuringLCP",
4988  "Test that we can execute the restart RestartRandomNode loop\n"\
4989  "number of times"){
4990  INITIALIZER(runCheckAllNodesStarted);
4991  INITIALIZER(runLoadTable);
4992  STEP(runRestarts);
4993  STEP(runPkUpdateUntilStopped);
4994  STEP(runScanUpdateUntilStopped);
4995  FINALIZER(runScanReadVerify);
4996  FINALIZER(runClearTable);
4997 }
4998 TESTCASE("StopOnError",
4999  "Test StopOnError. A node that has StopOnError set to false "\
5000  "should restart automatically when an error occurs"){
5001  INITIALIZER(runCheckAllNodesStarted);
5002  INITIALIZER(runLoadTable);
5003  STEP(runRestarts);
5004  FINALIZER(runScanReadVerify);
5005  FINALIZER(runClearTable);
5006 }
5007 TESTCASE("CommittedRead",
5008  "Test committed read"){
5009  INITIALIZER(runLoadTable);
5010  STEP(runDirtyRead);
5011  FINALIZER(runClearTable);
5012 }
5013 TESTCASE("LateCommit",
5014  "Test commit after node failure"){
5015  INITIALIZER(runLoadTable);
5016  STEP(runLateCommit);
5017  FINALIZER(runClearTable);
5018 }
5019 TESTCASE("Bug15587",
5020  "Test bug with NF during NR"){
5021  INITIALIZER(runLoadTable);
5022  STEP(runScanUpdateUntilStopped);
5023  STEP(runBug15587);
5024  FINALIZER(runClearTable);
5025 }
5026 TESTCASE("Bug15632",
5027  "Test bug with NF during NR"){
5028  INITIALIZER(runLoadTable);
5029  STEP(runBug15632);
5030  FINALIZER(runClearTable);
5031 }
5032 TESTCASE("Bug15685",
5033  "Test bug with NF during abort"){
5034  STEP(runBug15685);
5035  FINALIZER(runClearTable);
5036 }
5037 TESTCASE("Bug16772",
5038  "Test bug with restarting before NF handling is complete"){
5039  STEP(runBug16772);
5040 }
5041 TESTCASE("Bug18414",
5042  "Test bug with NF during NR"){
5043  INITIALIZER(runLoadTable);
5044  STEP(runBug18414);
5045  FINALIZER(runClearTable);
5046 }
5047 TESTCASE("Bug18612",
5048  "Test bug with partitioned clusters"){
5049  INITIALIZER(runLoadTable);
5050  STEP(runBug18612);
5051  FINALIZER(runClearTable);
5052 }
5053 TESTCASE("Bug18612SR",
5054  "Test bug with partitioned clusters"){
5055  INITIALIZER(runLoadTable);
5056  STEP(runBug18612SR);
5057  FINALIZER(runClearTable);
5058 }
5059 TESTCASE("Bug20185",
5060  ""){
5061  INITIALIZER(runLoadTable);
5062  STEP(runBug20185);
5063  FINALIZER(runClearTable);
5064 }
5065 TESTCASE("Bug24543", "")
5066 {
5067  INITIALIZER(runBug24543);
5068 }
5069 TESTCASE("Bug21271",
5070  ""){
5071  INITIALIZER(runLoadTable);
5072  STEP(runBug21271);
5073  STEP(runPkUpdateUntilStopped);
5074  FINALIZER(runClearTable);
5075 }
5076 TESTCASE("Bug24717", ""){
5077  INITIALIZER(runBug24717);
5078 }
5079 TESTCASE("Bug25364", ""){
5080  INITIALIZER(runBug25364);
5081 }
5082 TESTCASE("Bug25468", ""){
5083  INITIALIZER(runBug25468);
5084 }
5085 TESTCASE("Bug25554", ""){
5086  INITIALIZER(runBug25554);
5087 }
5088 TESTCASE("Bug25984", ""){
5089  INITIALIZER(runBug25984);
5090 }
5091 TESTCASE("Bug26457", ""){
5092  INITIALIZER(runBug26457);
5093 }
5094 TESTCASE("Bug26481", ""){
5095  INITIALIZER(runBug26481);
5096 }
5097 TESTCASE("Bug26450", ""){
5098  INITIALIZER(runLoadTable);
5099  INITIALIZER(runBug26450);
5100 }
5101 TESTCASE("Bug27003", ""){
5102  INITIALIZER(runBug27003);
5103 }
5104 TESTCASE("Bug27283", ""){
5105  INITIALIZER(runBug27283);
5106 }
5107 TESTCASE("Bug27466", ""){
5108  INITIALIZER(runBug27466);
5109 }
5110 TESTCASE("Bug28023", ""){
5111  INITIALIZER(runBug28023);
5112 }
5113 TESTCASE("Bug28717", ""){
5114  INITIALIZER(runBug28717);
5115 }
5116 TESTCASE("Bug31980", ""){
5117  INITIALIZER(runBug31980);
5118 }
5119 TESTCASE("Bug29364", ""){
5120  INITIALIZER(runBug29364);
5121 }
5122 TESTCASE("GCP", ""){
5123  INITIALIZER(runLoadTable);
5124  STEP(runGCP);
5125  STEP(runScanUpdateUntilStopped);
5126  FINALIZER(runClearTable);
5127 }
5128 TESTCASE("CommitAck", ""){
5129  INITIALIZER(runCommitAck);
5130  FINALIZER(runClearTable);
5131 }
5132 TESTCASE("Bug32160", ""){
5133  INITIALIZER(runBug32160);
5134 }
5135 TESTCASE("pnr", "Parallel node restart")
5136 {
5137  TC_PROPERTY("ScanUpdateNoRowCountCheck", 1);
5138  INITIALIZER(runLoadTable);
5139  INITIALIZER(runCreateBigTable);
5140  STEP(runScanUpdateUntilStopped);
5141  STEP(runDeleteInsertUntilStopped);
5142  STEP(runPnr);
5143  FINALIZER(runClearTable);
5144  FINALIZER(runDropBigTable);
5145 }
5146 TESTCASE("pnr_lcp", "Parallel node restart")
5147 {
5148  TC_PROPERTY("LCP", 1);
5149  TC_PROPERTY("ScanUpdateNoRowCountCheck", 1);
5150  INITIALIZER(runLoadTable);
5151  INITIALIZER(runCreateBigTable);
5152  STEP(runScanUpdateUntilStopped);
5153  STEP(runDeleteInsertUntilStopped);
5154  STEP(runPnr);
5155  FINALIZER(runClearTable);
5156  FINALIZER(runDropBigTable);
5157 }
5158 TESTCASE("Bug32922", ""){
5159  INITIALIZER(runBug32922);
5160 }
5161 TESTCASE("Bug34216", ""){
5162  INITIALIZER(runCheckAllNodesStarted);
5163  INITIALIZER(runLoadTable);
5164  STEP(runBug34216);
5165  FINALIZER(runClearTable);
5166 }
5167 TESTCASE("mixedmultiop", ""){
5168  TC_PROPERTY("MULTI_OP", 5);
5169  INITIALIZER(runCheckAllNodesStarted);
5170  INITIALIZER(runLoadTable);
5171  STEP(runNF_commit);
5172  STEP(runPkUpdateUntilStopped);
5173  STEP(runPkUpdateUntilStopped);
5174  FINALIZER(runClearTable);
5175 }
5176 TESTCASE("Bug34702", ""){
5177  INITIALIZER(runBug34702);
5178 }
5179 TESTCASE("MNF", ""){
5180  INITIALIZER(runLoadTable);
5181  STEP(runMNF);
5182  STEP(runScanUpdateUntilStopped);
5183 }
5184 TESTCASE("Bug36199", ""){
5185  INITIALIZER(runBug36199);
5186 }
5187 TESTCASE("Bug36246", ""){
5188  INITIALIZER(runLoadTable);
5189  STEP(runBug36246);
5190  VERIFIER(runClearTable);
5191 }
5192 TESTCASE("Bug36247", ""){
5193  INITIALIZER(runLoadTable);
5194  STEP(runBug36247);
5195  VERIFIER(runClearTable);
5196 }
5197 TESTCASE("Bug36276", ""){
5198  INITIALIZER(runLoadTable);
5199  STEP(runBug36276);
5200  VERIFIER(runClearTable);
5201 }
5202 TESTCASE("Bug36245", ""){
5203  INITIALIZER(runLoadTable);
5204  STEP(runBug36245);
5205  VERIFIER(runClearTable);
5206 }
5207 TESTCASE("NF_Hammer", ""){
5208  TC_PROPERTY("Sleep0", 9000);
5209  TC_PROPERTY("Sleep1", 3000);
5210  TC_PROPERTY("Rand", 1);
5211  INITIALIZER(runLoadTable);
5212  STEPS(runHammer, 25);
5213  STEP(runRestarter);
5214  VERIFIER(runClearTable);
5215 }
5216 TESTCASE("Bug41295", "")
5217 {
5218  TC_PROPERTY("Threads", 25);
5219  INITIALIZER(runLoadTable);
5220  STEPS(runMixedLoad, 25);
5221  STEP(runBug41295);
5222  FINALIZER(runClearTable);
5223 }
5224 TESTCASE("Bug41469", ""){
5225  INITIALIZER(runLoadTable);
5226  STEP(runBug41469);
5227  STEP(runScanUpdateUntilStopped);
5228  FINALIZER(runClearTable);
5229 }
5230 TESTCASE("Bug42422", ""){
5231  INITIALIZER(runBug42422);
5232 }
5233 TESTCASE("Bug43224", ""){
5234  INITIALIZER(runBug43224);
5235 }
5236 TESTCASE("Bug58453", "")
5237 {
5238  INITIALIZER(runBug58453);
5239 }
5240 TESTCASE("Bug43888", ""){
5241  INITIALIZER(runBug43888);
5242 }
5243 TESTCASE("Bug44952",
5244  "Test that we can execute the restart RestartNFDuringNR loop\n" \
5245  "number of times"){
5246  INITIALIZER(runCheckAllNodesStarted);
5247  INITIALIZER(runLoadTable);
5248  STEP(runBug44952);
5249  STEP(runPkUpdateUntilStopped);
5250  STEP(runScanUpdateUntilStopped);
5251  FINALIZER(runScanReadVerify);
5252  FINALIZER(runClearTable);
5253 }
5254 TESTCASE("Bug48474", "")
5255 {
5256  INITIALIZER(runLoadTable);
5257  INITIALIZER(initBug48474);
5258  STEP(runBug48474);
5259  STEP(runScanUpdateUntilStopped);
5260  FINALIZER(cleanupBug48474);
5261 }
5262 TESTCASE("MixReadUnlockRestart",
5263  "Run mixed read+unlock and update transactions"){
5264  INITIALIZER(runCheckAllNodesStarted);
5265  INITIALIZER(runLoadTable);
5266  STEP(runPkReadPkUpdateUntilStopped);
5267  STEP(runPkReadPkUpdatePkUnlockUntilStopped);
5268  STEP(runPkReadPkUpdatePkUnlockUntilStopped);
5269  STEP(runRestarter);
5270  FINALIZER(runClearTable);
5271 }
5272 TESTCASE("Bug56044", "")
5273 {
5274  INITIALIZER(runBug56044);
5275 }
5276 TESTCASE("Bug57767", "")
5277 {
5278  INITIALIZER(runLoadTable);
5279  INITIALIZER(runBug57767)
5280 }
5281 TESTCASE("Bug57522", "")
5282 {
5283  INITIALIZER(runBug57522);
5284 }
5285 TESTCASE("ForceStopAndRestart", "Test restart and stop -with force flag")
5286 {
5287  STEP(runForceStopAndRestart);
5288 }
5289 TESTCASE("ClusterSplitLatency",
5290  "Test behaviour of 2-replica cluster with latency between halves")
5291 {
5292  TC_PROPERTY("DynamicOrder", Uint32(9));
5293  INITIALIZER(runRestartToDynamicOrder);
5294  INITIALIZER(analyseDynamicOrder);
5295  INITIALIZER(runSplitLatency25PctFail);
5296 }
5297 NDBT_TESTSUITE_END(testNodeRestart);
5298 
5299 int main(int argc, const char** argv){
5300  ndb_init();
5301  NDBT_TESTSUITE_INSTANCE(testNodeRestart);
5302 #if 0
5303  // It might be interesting to have longer defaults for num
5304  // loops in this test
5305  // Just performing 100 node restarts would not be enough?
5306  // We can have initialisers in the NDBT_Testcase class like
5307  // this...
5308  testNodeRestart.setDefaultLoops(1000);
5309 #endif
5310  return testNodeRestart.execute(argc, argv);
5311 }