testMgmd.cpp

/*
   Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.


   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
 */

#include <NDBT.hpp>
#include <NDBT_Test.hpp>
#include <portlib/NdbDir.hpp>
#include "ConfigFactory.hpp"
#include <NdbMgmd.hpp>
#include <NdbProcess.hpp>
#include <NDBT_Find.hpp>
#include <NDBT_Workingdir.hpp>

static bool file_exists(const char* path, Uint32 timeout = 1)
{
  g_info << "File '" << path << "' ";
  for (Uint32 i = 0; i < 10 * timeout; i++)
  {
    if (access(path, F_OK) == 0)
    {
      g_info << "exists" << endl;
      return true;
    }
    if (i == 0)
    {
      g_info << "does not exist, retrying...";
    }
    NdbSleep_MilliSleep(100);
  }
  g_info << "does not exist" << endl;
  return false;
}

// Util function that concatenate strings to form a path

static BaseString path(const char* first, ...)
{
  BaseString path;
  path.assign(first);

  const char* str;
  va_list args;
  va_start(args, first);
  while ((str = va_arg(args, const char*)) != NULL)
  {
    path.appfmt("%s%s", DIR_SEPARATOR, str);
  }
  va_end(args);
  return path;
}

class Mgmd
{
  NdbProcess* m_proc;
  int m_nodeid;
  BaseString m_name;
  BaseString m_exe;
  NdbMgmd m_mgmd_client;

  Mgmd(const Mgmd& other); // Not implemented
public:

  Mgmd(int nodeid) :
  m_proc(NULL),
  m_nodeid(nodeid)
  {
    m_name.assfmt("ndb_mgmd_%d", nodeid);

    NDBT_find_ndb_mgmd(m_exe);
  }

  ~Mgmd()
  {
    if (m_proc)
    {
      //stop the proces
      stop();
    }

  }

  const char* name(void) const
  {
    return m_name.c_str();
  }

  const char* exe(void) const
  {
    return m_exe.c_str();
  }

  bool start(const char* working_dir, NdbProcess::Args& args)
  {
    g_info << "Starting " << name() << " ";
    for (unsigned i = 0; i < args.args().size(); i++)
      g_info << args.args()[i].c_str() << " ";
    g_info << endl;

    m_proc = NdbProcess::create(name(),
                                exe(),
                                working_dir,
                                args);
    return (m_proc != NULL);
  }

  bool start_from_config_ini(const char* working_dir,
                             const char* first_extra_arg = NULL, ...)
  {
    NdbProcess::Args args;
    args.add("--configdir=.");
    args.add("-f config.ini");
    args.add("--ndb-nodeid=", m_nodeid);
    args.add("--nodaemon");
    args.add("--log-name=", name());
    args.add("--verbose");

    if (first_extra_arg)
    {
      // Append any extra args
      va_list extra_args;
      const char* str = first_extra_arg;
      va_start(extra_args, first_extra_arg);
      do
      {
        args.add(str);
      } while ((str = va_arg(extra_args, const char*)) != NULL);
      va_end(extra_args);
    }

    return start(working_dir, args);
  }

  bool start(const char* working_dir,
             const char* first_extra_arg = NULL, ...)
  {
    NdbProcess::Args args;
    args.add("--configdir=.");
    args.add("--ndb-nodeid=", m_nodeid);
    args.add("--nodaemon");
    args.add("--log-name=", name());
    args.add("--verbose");

    if (first_extra_arg)
    {
      // Append any extra args
      va_list extra_args;
      const char* str = first_extra_arg;
      va_start(extra_args, first_extra_arg);
      do
      {
        args.add(str);
      } while ((str = va_arg(extra_args, const char*)) != NULL);
      va_end(extra_args);
    }

    return start(working_dir, args);
  }

  bool stop(void)
  {
    g_info << "Stopping " << name() << endl;

    // Diconnect and close our "builtin" client
    m_mgmd_client.close();

    if (m_proc == 0 || !m_proc->stop())
    {
      fprintf(stderr, "Failed to stop process %s\n", name());
      return false; // Can't kill with -9 -> fatal error
    }
    int ret;
    if (!m_proc->wait(ret, 300))
    {
      fprintf(stderr, "Failed to wait for process %s\n", name());
      return false; // Can't wait after kill with -9 -> fatal error
    }

    if (ret != 9)
    {
      fprintf(stderr, "stop ret: %u\n", ret);
      return false; // Can't wait after kill with -9 -> fatal error
    }

    delete m_proc;
    m_proc = 0;

    return true;

  }

  bool wait(int& ret, int timeout = 300)
  {
    g_info << "Waiting for " << name() << endl;

    if (m_proc == 0 || !m_proc->wait(ret, timeout))
    {
      fprintf(stderr, "Failed to wait for process %s\n", name());
      return false;
    }
    delete m_proc;
    m_proc = 0;

    return true;

  }

  const BaseString connectstring(const Properties& config)
  {
    const char* hostname;
    if (!get_section_string(config, m_name.c_str(), "HostName", &hostname))
      return false;

    Uint32 port;
    if (!get_section_uint32(config, m_name.c_str(), "PortNumber", &port))
      return false;

    BaseString constr;
    constr.assfmt("%s:%d", hostname, port);
    return constr;
  }

  bool connect(const Properties& config,
               int num_retries = 60, int retry_delay_in_seconds = 1)
  {
    BaseString constr = connectstring(config);
    g_info << "Connecting to " << name() << " @ " << constr.c_str() << endl;

    return m_mgmd_client.connect(constr.c_str(),
                                 num_retries,
                                 retry_delay_in_seconds);
  }

  bool wait_confirmed_config(int timeout = 30)
  {
    if (!m_mgmd_client.is_connected())
    {
      g_err << "wait_confirmed_config: not connected!" << endl;
      return false;
    }

    int retries = 0;
    Config conf;
    while (!m_mgmd_client.get_config(conf))
    {
      retries++;

      if (retries == timeout * 10)
      {
        g_err << "wait_confirmed_config: Failed to get config within "
                << timeout << " seconds" << endl;
        return false;
      }

      g_err << "Failed to get config, sleeping" << endl;
      NdbSleep_MilliSleep(100);

    }
    g_info << "wait_confirmed_config: ok" << endl;
    return true;

  }

  NdbMgmHandle handle() { return m_mgmd_client.handle(); }

private:

  bool get_section_string(const Properties& config,
                          const char* section_name,
                          const char* key,
                          const char** value) const
  {
    const Properties* section;
    if (!config.get(section_name, &section))
      return false;

    if (!section->get(key, value))
      return false;
    return true;
  }

  bool get_section_uint32(const Properties& config,
                          const char* section_name,
                          const char* key,
                          Uint32* value) const
  {
    const Properties* section;
    if (!config.get(section_name, &section))
      return false;

    if (!section->get(key, value))
      return false;
    return true;
  }

};

class MgmdProcessList : public Vector<Mgmd*>
{
public:

  ~MgmdProcessList()
  {
    // Delete and thus stop the mgmd(s)
    for (unsigned i = 0; i < size(); i++)
    {
      Mgmd* mgmd = this->operator[](i);
      delete mgmd;
    }
    //  delete this->[i];
    clear();
  }
};


#define CHECK(x)                                            \
  if (!(x)) {                                               \
    fprintf(stderr, "CHECK(" #x ") failed at line: %d\n", \
            __LINE__);                                      \
     return NDBT_FAILED;                                    \
  }

int runTestBasic2Mgm(NDBT_Context* ctx, NDBT_Step* step)
{
  NDBT_Workingdir wd("test_mgmd"); // temporary working directory

  // Create config.ini
  Properties config = ConfigFactory::create(2);
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));
  // Start ndb_mgmd(s)
  MgmdProcessList mgmds;
  for (int i = 1; i <= 2; i++)
  {
    Mgmd* mgmd = new Mgmd(i);
    mgmds.push_back(mgmd);
    CHECK(mgmd->start_from_config_ini(wd.path()));
  }

  // Connect the ndb_mgmd(s)
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->connect(config));

  // wait for confirmed config
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->wait_confirmed_config());

  // Check binary config files created
  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.1",
                         NULL).c_str()));
  CHECK(file_exists(path(wd.path(),
                         "ndb_2_config.bin.1",
                         NULL).c_str()));

  // Stop the ndb_mgmd(s)
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->stop());

  // Start up the mgmd(s) again from config.bin
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->start_from_config_ini(wd.path()));

  // Connect the ndb_mgmd(s)
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->connect(config));

  // check ndb_X_config.bin.1 still exists but not ndb_X_config.bin.2
  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.1",
                         NULL).c_str()));
  CHECK(file_exists(path(wd.path(),
                         "ndb_2_config.bin.1",
                         NULL).c_str()));

  CHECK(!file_exists(path(wd.path(),
                          "ndb_1_config.bin.2",
                          NULL).c_str()));
  CHECK(!file_exists(path(wd.path(),
                          "ndb_2_config.bin.2",
                          NULL).c_str()));

  return NDBT_OK;

}

int runTestBug45495(NDBT_Context* ctx, NDBT_Step* step)
{
  NDBT_Workingdir wd("test_mgmd"); // temporary working directory

  g_err << "** Create config.ini" << endl;
  Properties config = ConfigFactory::create(2);
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));
  // Start ndb_mgmd(s)
  MgmdProcessList mgmds;
  for (int i = 1; i <= 2; i++)
  {
    Mgmd* mgmd = new Mgmd(i);
    mgmds.push_back(mgmd);
    CHECK(mgmd->start_from_config_ini(wd.path()));
  }

  // Connect the ndb_mgmd(s)
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->connect(config));

  // wait for confirmed config
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->wait_confirmed_config());

  // Check binary config files created
  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.1",
                         NULL).c_str()));
  CHECK(file_exists(path(wd.path(),
                         "ndb_2_config.bin.1",
                         NULL).c_str()));

  g_err << "** Restart one ndb_mgmd at a time --reload + --initial" << endl;
  for (unsigned i = 0; i < mgmds.size(); i++)
  {
    CHECK(mgmds[i]->stop());
    CHECK(mgmds[i]->start_from_config_ini(wd.path(),
                                          "--reload", "--initial", NULL));
    CHECK(mgmds[i]->connect(config));
    CHECK(mgmds[i]->wait_confirmed_config());

    // check ndb_X_config.bin.1 still exists but not ndb_X_config.bin.2
    CHECK(file_exists(path(wd.path(),
                           "ndb_1_config.bin.1",
                           NULL).c_str()));
    CHECK(file_exists(path(wd.path(),
                           "ndb_2_config.bin.1",
                           NULL).c_str()));

    CHECK(!file_exists(path(wd.path(),
                            "ndb_1_config.bin.2",
                            NULL).c_str()));
    CHECK(!file_exists(path(wd.path(),
                            "ndb_2_config.bin.2",
                            NULL).c_str()));
  }

  g_err << "** Restart one ndb_mgmd at a time --initial" << endl;
  for (unsigned i = 0; i < mgmds.size(); i++)
  {
    CHECK(mgmds[i]->stop());
    CHECK(mgmds[i]->start_from_config_ini(wd.path(),
                                          "--initial", NULL));
    CHECK(mgmds[i]->connect(config));
    CHECK(mgmds[i]->wait_confirmed_config());

    // check ndb_X_config.bin.1 still exists but not ndb_X_config.bin.2
    CHECK(file_exists(path(wd.path(),
                           "ndb_1_config.bin.1",
                           NULL).c_str()));
    CHECK(file_exists(path(wd.path(),
                           "ndb_2_config.bin.1",
                           NULL).c_str()));

    CHECK(!file_exists(path(wd.path(),
                            "ndb_1_config.bin.2",
                            NULL).c_str()));
    CHECK(!file_exists(path(wd.path(),
                            "ndb_2_config.bin.2",
                            NULL).c_str()));
  }

  g_err << "** Create config2.ini" << endl;
  CHECK(ConfigFactory::put(config, "ndb_mgmd", 1, "ArbitrationDelay", 100));
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config2.ini",
                                             NULL).c_str()));

  g_err << "** Restart one ndb_mgmd at a time --initial should not work" << endl;
  for (unsigned i = 0; i < mgmds.size(); i++)
  {
    CHECK(mgmds[i]->stop());
    // Start from config2.ini
    CHECK(mgmds[i]->start_from_config_ini(wd.path(),
                                          "-f config2.ini",
                                          "--initial", NULL));

    // Wait for mgmd to exit and check return status
    int ret;
    CHECK(mgmds[i]->wait(ret));
    CHECK(ret == 1);

    // check config files exist only for the still running mgmd(s)
    for (unsigned j = 0; j < mgmds.size(); j++)
    {
      BaseString tmp;
      tmp.assfmt("ndb_%d_config.bin.1", j+1);
      CHECK(file_exists(path(wd.path(),
                             tmp.c_str(),
                             NULL).c_str()) == (j != i));
    }

    // Start from config.ini again
    CHECK(mgmds[i]->start_from_config_ini(wd.path(),
                                          "--initial",
                                          "--reload",
                                          NULL));
    CHECK(mgmds[i]->connect(config));
    CHECK(mgmds[i]->wait_confirmed_config());
  }

  g_err << "** Reload from config2.ini" << endl;
  for (unsigned i = 0; i < mgmds.size(); i++)
  {
    CHECK(mgmds[i]->stop());
    // Start from config2.ini
    CHECK(mgmds[i]->start_from_config_ini(wd.path(),
                                          "-f config2.ini",
                                          "--reload", NULL));
    CHECK(mgmds[i]->connect(config));
    CHECK(mgmds[i]->wait_confirmed_config());
  }

  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.1",
                         NULL).c_str()));
  CHECK(file_exists(path(wd.path(),
                         "ndb_2_config.bin.1",
                         NULL).c_str()));

  Uint32 timeout = 30;
  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.2",
                         NULL).c_str(), timeout));
  CHECK(file_exists(path(wd.path(),
                         "ndb_2_config.bin.2",
                         NULL).c_str(), timeout));

  g_err << "** Reload mgmd initial(from generation=2)" << endl;
  for (unsigned i = 0; i < mgmds.size(); i++)
  {
    CHECK(mgmds[i]->stop());
    CHECK(mgmds[i]->start_from_config_ini(wd.path(),
                                          "-f config2.ini",
                                          "--reload", "--initial", NULL));

    CHECK(mgmds[i]->connect(config));
    CHECK(mgmds[i]->wait_confirmed_config());

     // check config files exist
    for (unsigned j = 0; j < mgmds.size(); j++)
    {
      BaseString tmp;
      tmp.assfmt("ndb_%d_config.bin.1", j+1);
      CHECK(file_exists(path(wd.path(),
                             tmp.c_str(),
                             NULL).c_str()) == (i < j));

      tmp.assfmt("ndb_%d_config.bin.2", j+1);
      CHECK(file_exists(path(wd.path(),
                             tmp.c_str(),
                             NULL).c_str(),
                        timeout));
    }
  }

  return NDBT_OK;
}



int runTestBug42015(NDBT_Context* ctx, NDBT_Step* step)
{
  NDBT_Workingdir wd("test_mgmd"); // temporary working directory

  g_err << "** Create config.ini" << endl;
  Properties config = ConfigFactory::create(2);
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));

  MgmdProcessList mgmds;
  // Start ndb_mgmd 1 from config.ini
  Mgmd* mgmd = new Mgmd(1);
  mgmds.push_back(mgmd);
  CHECK(mgmd->start_from_config_ini(wd.path()));

  // Start ndb_mgmd 2 by fetching from first
  Mgmd* mgmd2 = new Mgmd(2);
  mgmds.push_back(mgmd2);
  CHECK(mgmd2->start(wd.path(),
                     "--ndb-connectstring",
                     mgmd->connectstring(config).c_str(),
                     NULL));

  // Connect the ndb_mgmd(s)
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->connect(config));

  // wait for confirmed config
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->wait_confirmed_config());

  // Check binary config files created
  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.1",
                         NULL).c_str()));
  CHECK(file_exists(path(wd.path(),
                         "ndb_2_config.bin.1",
                         NULL).c_str()));

  return NDBT_OK;

}

/* Test for bug 53008:  --skip-config-cache */
int runTestNoConfigCache(NDBT_Context* ctx, NDBT_Step* step)
{
  NDBT_Workingdir wd("test_mgmd"); // temporary working directory
  
  g_err << "** Create config.ini" << endl;
  Properties config = ConfigFactory::create();
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));
  
  // Start ndb_mgmd  from config.ini
  Mgmd* mgmd = new Mgmd(1);
  CHECK(mgmd->start_from_config_ini(wd.path(), "--skip-config-cache", NULL));
     
  // Connect the ndb_mgmd(s)
  CHECK(mgmd->connect(config));
  
  // wait for confirmed config
  CHECK(mgmd->wait_confirmed_config());
  
  // Check binary config files *not* created
  bool bin_conf_file = file_exists(path(wd.path(),
                                        "ndb_1_config.bin.1", 
                                        NULL).c_str());
  CHECK(bin_conf_file == false);
  
  mgmd->stop();  
  return NDBT_OK;
}  


int runTestNowaitNodes(NDBT_Context* ctx, NDBT_Step* step)
{
  MgmdProcessList mgmds;
  NDBT_Workingdir wd("test_mgmd"); // temporary working directory

  // Create config.ini
  Properties config = ConfigFactory::create(2);
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));
  // Start first ndb_mgmd
  Mgmd* mgmd1 = new Mgmd(1);
  {
    mgmds.push_back(mgmd1);
    CHECK(mgmd1->start_from_config_ini(wd.path(),
                                    "--initial",
                                       "--nowait-nodes=2",
                                       NULL));

    // Connect the ndb_mgmd
    CHECK(mgmd1->connect(config));

    // wait for confirmed config
    CHECK(mgmd1->wait_confirmed_config());

    // Check binary config file created
    CHECK(file_exists(path(wd.path(),
                           "ndb_1_config.bin.1",
                           NULL).c_str()));
  }

  // Start second ndb_mgmd
  {
    Mgmd* mgmd2 = new Mgmd(2);
    mgmds.push_back(mgmd2);
    CHECK(mgmd2->start_from_config_ini(wd.path(),
                                       "--initial",
                                       NULL));

    // Connect the ndb_mgmd
    CHECK(mgmd2->connect(config));

    // wait for confirmed config
    CHECK(mgmd2->wait_confirmed_config());

    // Check binary config file created
    CHECK(file_exists(path(wd.path(),
                           "ndb_2_config.bin.1",
                           NULL).c_str()));

  }

  // Create new config.ini
  g_err << "** Create config2.ini" << endl;
  CHECK(ConfigFactory::put(config, "ndb_mgmd", 1, "ArbitrationDelay", 100));
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config2.ini",
                                             NULL).c_str()));

  g_err << "** Reload second mgmd from config2.ini" << endl;
  {
    Mgmd* mgmd2 = mgmds[1];
    CHECK(mgmd2->stop());
    // Start from config2.ini
    CHECK(mgmd2->start_from_config_ini(wd.path(),
                                       "-f config2.ini",
                                       "--reload", NULL));
    CHECK(mgmd2->connect(config));
    CHECK(mgmd1->wait_confirmed_config());
    CHECK(mgmd2->wait_confirmed_config());

    CHECK(file_exists(path(wd.path(),
                           "ndb_1_config.bin.1",
                           NULL).c_str()));
    CHECK(file_exists(path(wd.path(),
                           "ndb_2_config.bin.1",
                           NULL).c_str()));

    // Both ndb_mgmd(s) should have reloaded and new binary config exist
    CHECK(file_exists(path(wd.path(),
                           "ndb_1_config.bin.2",
                           NULL).c_str()));
    CHECK(file_exists(path(wd.path(),
                           "ndb_2_config.bin.2",
                           NULL).c_str()));
  }

  // Stop the ndb_mgmd(s)
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->stop());

  return NDBT_OK;
}


int runTestNowaitNodes2(NDBT_Context* ctx, NDBT_Step* step)
{
  int ret;
  NDBT_Workingdir wd("test_mgmd"); // temporary working directory

  // Create config.ini
  Properties config = ConfigFactory::create(2);
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));

  g_err << "** Start mgmd1 from config.ini" << endl;
  MgmdProcessList mgmds;
  Mgmd* mgmd1 = new Mgmd(1);
  mgmds.push_back(mgmd1);
  CHECK(mgmd1->start_from_config_ini(wd.path(),
                                     "--initial",
                                     "--nowait-nodes=1-255",
                                     NULL));
  CHECK(mgmd1->connect(config));
  CHECK(mgmd1->wait_confirmed_config());

  // check config files exist
  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.1",
                         NULL).c_str()));

  g_err << "** Create config2.ini" << endl;
  CHECK(ConfigFactory::put(config, "ndb_mgmd", 1, "ArbitrationDelay", 100));
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config2.ini",
                                             NULL).c_str()));

  g_err << "** Start mgmd2 from config2.ini" << endl;
  Mgmd* mgmd2 = new Mgmd(2);
  mgmds.push_back(mgmd2);
  CHECK(mgmd2->start_from_config_ini(wd.path(),
                                     "-f config2.ini",
                                     "--initial",
                                     "--nowait-nodes=1-255",
                                     NULL));
  CHECK(mgmd2->wait(ret));
  CHECK(ret == 1);

  CHECK(mgmd1->stop());

  g_err << "** Start mgmd2 again from config2.ini" << endl;
  CHECK(mgmd2->start_from_config_ini(wd.path(),
                                     "-f config2.ini",
                                     "--initial",
                                     "--nowait-nodes=1-255",
                                     NULL));


  CHECK(mgmd2->connect(config));
  CHECK(mgmd2->wait_confirmed_config());

  // check config files exist
  CHECK(file_exists(path(wd.path(),
                         "ndb_2_config.bin.1",
                         NULL).c_str()));

  g_err << "** Start mgmd1 from config.ini, mgmd2 should shutdown" << endl;
  CHECK(mgmd1->start_from_config_ini(wd.path(),
                                     "--initial",
                                     "--nowait-nodes=1-255",
                                     NULL));
  CHECK(mgmd2->wait(ret));
  CHECK(ret == 1);

  CHECK(mgmd1->stop());

  return NDBT_OK;
}

int
runBug56844(NDBT_Context* ctx, NDBT_Step* step)
{
  NDBT_Workingdir wd("test_mgmd"); // temporary working directory

  g_err << "** Create config.ini" << endl;
  Properties config = ConfigFactory::create(2);
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));
  // Start ndb_mgmd(s)
  MgmdProcessList mgmds;
  for (int i = 1; i <= 2; i++)
  {
    Mgmd* mgmd = new Mgmd(i);
    mgmds.push_back(mgmd);
    CHECK(mgmd->start_from_config_ini(wd.path()));
  }

  // Connect the ndb_mgmd(s)
  for (unsigned i = 0; i < mgmds.size(); i++)
  {
    CHECK(mgmds[i]->connect(config));
  }

  // wait for confirmed config
  for (unsigned i = 0; i < mgmds.size(); i++)
  {
    CHECK(mgmds[i]->wait_confirmed_config());
  }

  // stop them
  for (unsigned i = 0; i < mgmds.size(); i++)
  {
    CHECK(mgmds[i]->stop());
  }

  // Check binary config files created
  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.1",
                         NULL).c_str()));
  CHECK(file_exists(path(wd.path(),
                         "ndb_2_config.bin.1",
                         NULL).c_str()));

  CHECK(ConfigFactory::put(config, "ndb_mgmd", 1, "ArbitrationDelay", 100));
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config2.ini",
                                             NULL).c_str()));
  Uint32 no = 2;
  int loops = ctx->getNumLoops();
  for (int l = 0; l < loops; l++, no++)
  {
    g_err << l << ": *** Reload from config.ini" << endl;
    for (unsigned i = 0; i < mgmds.size(); i++)
    {
      // Start from config2.ini
      CHECK(mgmds[i]->start_from_config_ini(wd.path(),
                                            (l & 1) == 1 ?
                                            "-f config.ini" :
                                            "-f config2.ini",
                                            "--reload", NULL));
    }
    for (unsigned i = 0; i < mgmds.size(); i++)
    {
      CHECK(mgmds[i]->connect(config));
      CHECK(mgmds[i]->wait_confirmed_config());
    }

    Uint32 timeout = 30;
    for (unsigned i = 0; i < mgmds.size(); i++)
    {
      BaseString p = path(wd.path(), "", NULL);
      p.appfmt("ndb_%u_config.bin.%u", i+1, no);
      g_err << "CHECK(" << p.c_str() << ")" << endl;
      CHECK(file_exists(p.c_str(), timeout));
    }

    for (unsigned i = 0; i < mgmds.size(); i++)
    {
      CHECK(mgmds[i]->stop());
    }
  }
  return NDBT_OK;
}

static bool
get_status(const char* connectstring,
           Properties& status)
{
  NdbMgmd ndbmgmd;
  if (!ndbmgmd.connect(connectstring))
    return false;

  Properties args;
  if (!ndbmgmd.call("get status", args,
                    "node status", status, NULL, true))
  {
    g_err << "fetch_mgmd_status: mgmd.call failed" << endl;
    return false;
  }
  return true;
}

static bool
value_equal(Properties& status,
            int nodeid, const char* name,
            const char* expected_value)
{
  const char* value;
  BaseString key;
  key.assfmt("node.%d.%s", nodeid, name);
  if (!status.get(key.c_str(), &value))
  {
    g_err << "value_equal: no value found for '" << name
          << "." << nodeid << "'" << endl;
    return false;
  }

  if (strcmp(value, expected_value))
  {
    g_err << "value_equal: found unexpected value: '" << value
          << "', expected: '" << expected_value << "'" <<endl;
    return false;
  }
  g_info << "'" << value << "'=='" << expected_value << "'" << endl;
  return true;
}

#include <ndb_version.h>

int runTestBug12352191(NDBT_Context* ctx, NDBT_Step* step)
{
  BaseString version;
  version.assfmt("%u", NDB_VERSION_D);
  BaseString mysql_version;
  mysql_version.assfmt("%u", NDB_MYSQL_VERSION_D);
  BaseString address("127.0.0.1");

  NDBT_Workingdir wd("test_mgmd"); // temporary working directory

  g_err << "** Create config.ini" << endl;
  Properties config = ConfigFactory::create(2);
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));

  MgmdProcessList mgmds;
  const int nodeid1 = 1;
  Mgmd* mgmd1 = new Mgmd(nodeid1);
  mgmds.push_back(mgmd1);

  const int nodeid2 = 2;
  Mgmd* mgmd2 = new Mgmd(nodeid2);
  mgmds.push_back(mgmd2);

  // Start first mgmd
  CHECK(mgmd1->start_from_config_ini(wd.path()));
  CHECK(mgmd1->connect(config));

  Properties status1;
  CHECK(get_status(mgmd1->connectstring(config).c_str(), status1));
  //status1.print();
  // Check status for own mgm node, always CONNECTED
  CHECK(value_equal(status1, nodeid1, "type", "MGM"));
  CHECK(value_equal(status1, nodeid1, "status", "CONNECTED"));
  CHECK(value_equal(status1, nodeid1, "version", version.c_str()));
  CHECK(value_equal(status1, nodeid1, "mysql_version", mysql_version.c_str()));
  CHECK(value_equal(status1, nodeid1, "address", address.c_str()));
  CHECK(value_equal(status1, nodeid1, "startphase", "0"));
  CHECK(value_equal(status1, nodeid1, "dynamic_id", "0"));
  CHECK(value_equal(status1, nodeid1, "node_group", "0"));
  CHECK(value_equal(status1, nodeid1, "connect_count", "0"));

  // Check status for other mgm node
  // not started yet -> NO_CONTACT, no address, no versions
  CHECK(value_equal(status1, nodeid2, "type", "MGM"));
  CHECK(value_equal(status1, nodeid2, "status", "NO_CONTACT"));
  CHECK(value_equal(status1, nodeid2, "version", "0"));
  CHECK(value_equal(status1, nodeid2, "mysql_version", "0"));
  CHECK(value_equal(status1, nodeid2, "address", ""));
  CHECK(value_equal(status1, nodeid2, "startphase", "0"));
  CHECK(value_equal(status1, nodeid2, "dynamic_id", "0"));
  CHECK(value_equal(status1, nodeid2, "node_group", "0"));
  CHECK(value_equal(status1, nodeid2, "connect_count", "0"));

  // Start second mgmd
  CHECK(mgmd2->start_from_config_ini(wd.path()));
  CHECK(mgmd2->connect(config));

  // wait for confirmed config
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->wait_confirmed_config());

  Properties status2;
  CHECK(get_status(mgmd2->connectstring(config).c_str(), status2));
  //status2.print();
  // Check status for own mgm node, always CONNECTED
  CHECK(value_equal(status2, nodeid2, "type", "MGM"));
  CHECK(value_equal(status2, nodeid2, "status", "CONNECTED"));
  CHECK(value_equal(status2, nodeid2, "version", version.c_str()));
  CHECK(value_equal(status2, nodeid2, "mysql_version", mysql_version.c_str()));
  CHECK(value_equal(status2, nodeid2, "address", address.c_str()));
  CHECK(value_equal(status2, nodeid2, "startphase", "0"));
  CHECK(value_equal(status2, nodeid2, "dynamic_id", "0"));
  CHECK(value_equal(status2, nodeid2, "node_group", "0"));
  CHECK(value_equal(status2, nodeid2, "connect_count", "0"));

  // Check status for other mgm node
  // both started now -> CONNECTED, address and versions filled in
  CHECK(value_equal(status2, nodeid1, "type", "MGM"));
  CHECK(value_equal(status2, nodeid1, "status", "CONNECTED"));
  CHECK(value_equal(status2, nodeid1, "version", version.c_str()));
  CHECK(value_equal(status2, nodeid1, "mysql_version", mysql_version.c_str()));
  CHECK(value_equal(status2, nodeid1, "address", address.c_str()));
  CHECK(value_equal(status2, nodeid1, "startphase", "0"));
  CHECK(value_equal(status2, nodeid1, "dynamic_id", "0"));
  CHECK(value_equal(status2, nodeid1, "node_group", "0"));
  CHECK(value_equal(status2, nodeid1, "connect_count", "0"));

  Properties status3;
  CHECK(get_status(mgmd1->connectstring(config).c_str(), status3));
  //status3.print();
  // Check status for own mgm node, always CONNECTED
  CHECK(value_equal(status3, nodeid1, "type", "MGM"));
  CHECK(value_equal(status3, nodeid1, "status", "CONNECTED"));
  CHECK(value_equal(status3, nodeid1, "version", version.c_str()));
  CHECK(value_equal(status3, nodeid1, "mysql_version", mysql_version.c_str()));
  CHECK(value_equal(status3, nodeid1, "address", address.c_str()));
  CHECK(value_equal(status3, nodeid1, "startphase", "0"));
  CHECK(value_equal(status3, nodeid1, "dynamic_id", "0"));
  CHECK(value_equal(status3, nodeid1, "node_group", "0"));
  CHECK(value_equal(status3, nodeid1, "connect_count", "0"));

  // Check status for other mgm node
  // both started now -> CONNECTED, address and versions filled in
  CHECK(value_equal(status3, nodeid2, "type", "MGM"));
  CHECK(value_equal(status3, nodeid2, "status", "CONNECTED"));
  CHECK(value_equal(status3, nodeid2, "version", version.c_str()));
  CHECK(value_equal(status3, nodeid2, "mysql_version", mysql_version.c_str()));
  CHECK(value_equal(status3, nodeid2, "address", address.c_str()));
  CHECK(value_equal(status3, nodeid2, "startphase", "0"));
  CHECK(value_equal(status3, nodeid2, "dynamic_id", "0"));
  CHECK(value_equal(status3, nodeid2, "node_group", "0"));
  CHECK(value_equal(status3, nodeid2, "connect_count", "0"));

  return NDBT_OK;

}

int
runBug61607(NDBT_Context* ctx, NDBT_Step* step)
{
  NDBT_Workingdir wd("test_mgmd"); // temporary working directory

  // Create config.ini
  const int cnt_mgmd = 1;
  Properties config = ConfigFactory::create(cnt_mgmd);
  CHECK(ConfigFactory::write_config_ini(config,
                                        path(wd.path(),
                                             "config.ini",
                                             NULL).c_str()));
  // Start ndb_mgmd(s)
  MgmdProcessList mgmds;
  for (int i = 1; i <= cnt_mgmd; i++)
  {
    Mgmd* mgmd = new Mgmd(i);
    mgmds.push_back(mgmd);
    CHECK(mgmd->start_from_config_ini(wd.path()));
  }

  // Connect the ndb_mgmd(s)
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->connect(config));

  // wait for confirmed config
  for (unsigned i = 0; i < mgmds.size(); i++)
    CHECK(mgmds[i]->wait_confirmed_config());

  // Check binary config files created
  CHECK(file_exists(path(wd.path(),
                         "ndb_1_config.bin.1",
                         NULL).c_str()));

  int no_of_nodes = 0;
  int * node_ids = 0;
  int initialstart = 0;
  int nostart = 0;
  int abort = 0;
  int force = 0;
  int need_disconnect = 0;
  int res = ndb_mgm_restart4(mgmds[0]->handle(), no_of_nodes, node_ids,
                             initialstart, nostart, abort, force,
                             &need_disconnect);


  return res == 0 ? NDBT_OK : NDBT_FAILED;
}

NDBT_TESTSUITE(testMgmd);
DRIVER(DummyDriver); /* turn off use of NdbApi */

TESTCASE("Basic2Mgm",
         "Basic test with two mgmd")
{
  INITIALIZER(runTestBasic2Mgm);
}

TESTCASE("Bug42015",
         "Test that mgmd can fetch configuration from another mgmd")
{
  INITIALIZER(runTestBug42015);
}
TESTCASE("NowaitNodes",
         "Test that one mgmd(of 2) can start alone with usage "
         "of --nowait-nodes, then start the second mgmd and it should join")
{
  INITIALIZER(runTestNowaitNodes);
}
TESTCASE("NowaitNodes2",
         "Test that one mgmd(of 2) can start alone with usage "
         "of --nowait-nodes, then start the second mgmd from different "
         "configuration and the one with lowest nodeid should shutdown")
{
  INITIALIZER(runTestNowaitNodes2);
}

TESTCASE("NoCfgCache",
         "Test that when an mgmd is started with --skip-config-cache, "
         "no ndb_xx_config.xx.bin file is created, but you can "
         "connect to the mgm node and retrieve the config.")
{
  INITIALIZER(runTestNoConfigCache);
}

TESTCASE("Bug45495",
         "Test that mgmd can be restarted in any order")
{
  INITIALIZER(runTestBug45495);
}

TESTCASE("Bug56844",
         "Test that mgmd can be reloaded in parallel")
{
  INITIALIZER(runBug56844);
}
TESTCASE("Bug12352191",
         "Test mgmd status for other mgmd")
{
  INITIALIZER(runTestBug12352191);
}
TESTCASE("Bug61607", "")
{
  INITIALIZER(runBug61607);
}

NDBT_TESTSUITE_END(testMgmd);

int main(int argc, const char** argv)
{
  ndb_init();
  NDBT_TESTSUITE_INSTANCE(testMgmd);
  testMgmd.setCreateTable(false);
  testMgmd.setRunAllTables(true);
  testMgmd.setConnectCluster(false);
  return testMgmd.execute(argc, argv);
}

template class Vector<Mgmd*>;