content: // Licensed to the Apache Software Foundation (ASF) under one
  // or more contributor license agreements.  See the NOTICE file
  // distributed with this work for additional information
  // regarding copyright ownership.  The ASF licenses this file
  // to you under the Apache License, Version 2.0 (the
  // "License"); you may not use this file except in compliance
  // with the License.  You may obtain a copy of the License at
  //
  //   http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing,
  // software distributed under the License is distributed on an
  // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, either express or implied.  See the License for the
  // specific language governing permissions and limitations
  // under the License.
  package com.cloud.server;
  
  import java.io.File;
  import java.io.FileInputStream;
  import java.io.FileNotFoundException;
  import java.io.FileOutputStream;
  import java.io.IOException;
  import java.nio.file.Files;
  import java.security.NoSuchAlgorithmException;
  import java.sql.PreparedStatement;
  import java.sql.ResultSet;
  import java.sql.SQLException;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.Properties;
  import java.util.Set;
  import java.util.UUID;
  
  import javax.crypto.KeyGenerator;
  import javax.crypto.SecretKey;
  import javax.inject.Inject;
  import javax.naming.ConfigurationException;
  
  import org.apache.cloudstack.config.ApiServiceConfiguration;
  import org.apache.cloudstack.framework.config.ConfigDepot;
  import org.apache.cloudstack.framework.config.ConfigDepotAdmin;
  import org.apache.cloudstack.framework.config.ConfigKey;
  import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
  import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
  import org.apache.commons.codec.binary.Base64;
  import org.apache.commons.lang3.StringUtils;
  
  import com.cloud.configuration.Config;
  import com.cloud.configuration.ConfigurationManager;
  import com.cloud.configuration.Resource;
  import com.cloud.configuration.Resource.ResourceOwnerType;
  import com.cloud.configuration.Resource.ResourceType;
  import com.cloud.configuration.ResourceCountVO;
  import com.cloud.configuration.dao.ResourceCountDao;
  import com.cloud.dc.DataCenter.NetworkType;
  import com.cloud.dc.DataCenterVO;
  import com.cloud.dc.HostPodVO;
  import com.cloud.dc.VlanVO;
  import com.cloud.dc.dao.DataCenterDao;
  import com.cloud.dc.dao.HostPodDao;
  import com.cloud.dc.dao.VlanDao;
  import com.cloud.domain.DomainVO;
  import com.cloud.domain.dao.DomainDao;
  import com.cloud.exception.InternalErrorException;
  import com.cloud.exception.InvalidParameterValueException;
  import com.cloud.network.Network;
  import com.cloud.network.Network.GuestType;
  import com.cloud.network.Network.Provider;
  import com.cloud.network.Network.Service;
  import com.cloud.network.Network.State;
  import com.cloud.network.Networks.BroadcastDomainType;
  import com.cloud.network.Networks.Mode;
  import com.cloud.network.Networks.TrafficType;
  import com.cloud.network.dao.NetworkDao;
  import com.cloud.network.dao.NetworkVO;
  import com.cloud.network.guru.ControlNetworkGuru;
  import com.cloud.network.guru.DirectPodBasedNetworkGuru;
  import com.cloud.network.guru.PodBasedNetworkGuru;
  import com.cloud.network.guru.PublicNetworkGuru;
  import com.cloud.network.guru.StorageNetworkGuru;
  import com.cloud.offering.NetworkOffering;
  import com.cloud.offering.NetworkOffering.Availability;
  import com.cloud.offerings.NetworkOfferingServiceMapVO;
  import com.cloud.offerings.NetworkOfferingVO;
  import com.cloud.offerings.dao.NetworkOfferingDao;
  import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
  import com.cloud.service.ServiceOfferingVO;
  import com.cloud.service.dao.ServiceOfferingDao;
  import com.cloud.storage.DiskOfferingVO;
  import com.cloud.storage.Storage.ProvisioningType;
  import com.cloud.storage.dao.DiskOfferingDao;
  import com.cloud.test.IPRangeConfig;
  import com.cloud.user.Account;
  import com.cloud.user.AccountVO;
  import com.cloud.user.User;
  import com.cloud.user.dao.AccountDao;
  import com.cloud.utils.Pair;
  import com.cloud.utils.PasswordGenerator;
  import com.cloud.utils.PropertiesUtil;
  import com.cloud.utils.component.ComponentLifecycle;
  import com.cloud.utils.component.ManagerBase;
  import com.cloud.utils.crypt.DBEncryptionUtil;
  import com.cloud.utils.db.DB;
  import com.cloud.utils.db.Transaction;
  import com.cloud.utils.db.TransactionCallbackNoReturn;
  import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn;
  import com.cloud.utils.db.TransactionLegacy;
  import com.cloud.utils.db.TransactionStatus;
  import com.cloud.utils.exception.CloudRuntimeException;
  import com.cloud.utils.net.NetUtils;
  import com.cloud.utils.script.Script;
  
  public class ConfigurationServerImpl extends ManagerBase implements ConfigurationServer {
  
      @Inject
      private ConfigurationDao _configDao;
      @Inject
      private DataCenterDao _zoneDao;
      @Inject
      private HostPodDao _podDao;
      @Inject
      private DiskOfferingDao _diskOfferingDao;
      @Inject
      private ServiceOfferingDao _serviceOfferingDao;
      @Inject
      private NetworkOfferingDao _networkOfferingDao;
      @Inject
      private DataCenterDao _dataCenterDao;
      @Inject
      private NetworkDao _networkDao;
      @Inject
      private VlanDao _vlanDao;
      @Inject
      private DomainDao _domainDao;
      @Inject
      private AccountDao _accountDao;
      @Inject
      private ResourceCountDao _resourceCountDao;
      @Inject
      private NetworkOfferingServiceMapDao _ntwkOfferingServiceMapDao;
      @Inject
      protected ConfigDepotAdmin _configDepotAdmin;
      @Inject
      protected ConfigDepot _configDepot;
      @Inject
      protected ConfigurationManager _configMgr;
      @Inject
      protected ManagementService _mgrService;
  
  
      public ConfigurationServerImpl() {
          setRunLevel(ComponentLifecycle.RUN_LEVEL_FRAMEWORK_BOOTSTRAP);
      }
  
      @Override
      public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
          try {
              persistDefaultValues();
              _configDepotAdmin.populateConfigurations();
          } catch (InternalErrorException | CloudRuntimeException e) {
              logger.error("Unhandled configuration exception: " + e.getMessage());
              throw new CloudRuntimeException("Unhandled configuration exception", e);
          }
          return true;
      }
  
      @Override
      public void persistDefaultValues() throws InternalErrorException {
  
          // Create system user and admin user
          saveUser();
  
          // Get init
          String init = _configDao.getValue("init");
  
          if (init == null || init.equals("false")) {
              logger.debug("ConfigurationServer is saving default values to the database.");
  
              // Save default Configuration Table values
              List<String> categories = Config.getCategories();
              for (String category : categories) {
                  // If this is not a premium environment, don't insert premium configuration values
                  if (!_configDao.isPremium() && category.equals("Premium")) {
                      continue;
                  }
  
                  List<Config> configs = Config.getConfigs(category);
                  for (Config c : configs) {
                      String name = c.key();
  
                      // if the config value already present in the db, don't insert it again
                      if (_configDao.findByName(name) != null) {
                          continue;
                      }
  
                      String instance = "DEFAULT";
                      String component = c.getComponent();
                      String value = c.getDefaultValue();
                      String description = c.getDescription();
                      ConfigurationVO configVO = new ConfigurationVO(category, instance, component, name, value, description);
                      configVO.setDefaultValue(value);
                      Pair<Long, Long> configGroupAndSubGroup = _configDepotAdmin.getConfigurationGroupAndSubGroupByName(name);
                      configVO.setGroupId(configGroupAndSubGroup.first());
                      configVO.setSubGroupId(configGroupAndSubGroup.second());
                      if (c.getKind() != null) {
                          configVO.setKind(c.getKind());
                      }
                      if (c.getOptions() != null) {
                          configVO.setOptions(c.getOptions());
                      }
  
                      _configDao.persist(configVO);
                  }
              }
  
              _configDao.update(Config.UseSecondaryStorageVm.key(), Config.UseSecondaryStorageVm.getCategory(), "true");
              logger.debug("ConfigurationServer made secondary storage vm required.");
  
              _configDao.update(Config.SecStorageEncryptCopy.key(), Config.SecStorageEncryptCopy.getCategory(), "false");
              logger.debug("ConfigurationServer made secondary storage copy encrypt set to false.");
  
              _configDao.update("secstorage.secure.copy.cert", "realhostip");
              logger.debug("ConfigurationServer made secondary storage copy use realhostip.");
  
              _configDao.update("user.password.encoders.exclude", "MD5,LDAP,PLAINTEXT");
              logger.debug("Configuration server excluded insecure encoders");
  
              _configDao.update("user.authenticators.exclude", "PLAINTEXT");
              logger.debug("Configuration server excluded plaintext authenticator");
  
              // Save default service offerings
              createServiceOffering(User.UID_SYSTEM, "Small Instance", 1, 512, 500, "Small Instance", ProvisioningType.THIN, false, false, null);
              createServiceOffering(User.UID_SYSTEM, "Medium Instance", 1, 1024, 1000, "Medium Instance", ProvisioningType.THIN, false, false, null);
              // Save default disk offerings
              createDefaultDiskOffering("Small", "Small Disk, 5 GB", ProvisioningType.THIN, 5, null, false, false);
              createDefaultDiskOffering("Medium", "Medium Disk, 20 GB", ProvisioningType.THIN, 20, null, false, false);
              createDefaultDiskOffering("Large", "Large Disk, 100 GB", ProvisioningType.THIN, 100, null, false, false);
              createDefaultDiskOffering("Large", "Large Disk, 100 GB", ProvisioningType.THIN, 100, null, false, false);
              createDefaultDiskOffering("Custom", "Custom Disk", ProvisioningType.THIN, 0, null, true, false);
  
              // Save the mount parent to the configuration table
              String mountParent = getMountParent();
              if (mountParent != null) {
                  _configDao.update(Config.MountParent.key(), Config.MountParent.getCategory(), mountParent);
                  logger.debug("ConfigurationServer saved \"" + mountParent + "\" as mount.parent.");
              } else {
                  logger.debug("ConfigurationServer could not detect mount.parent.");
              }
  
              String hostIpAdr = NetUtils.getDefaultHostIp();
              boolean needUpdateHostIp = true;
              if (hostIpAdr != null) {
                  Boolean devel = Boolean.valueOf(_configDao.getValue("developer"));
                  if (devel) {
                      String value = _configDao.getValue(ApiServiceConfiguration.ManagementServerAddresses.key());
                      if (value != null && !value.equals("localhost")) {
                          needUpdateHostIp = false;
                      }
                  }
  
                  if (needUpdateHostIp) {
                      _configDepot.createOrUpdateConfigObject(ApiServiceConfiguration.class.getSimpleName(), ApiServiceConfiguration.ManagementServerAddresses, hostIpAdr);
                      logger.debug("ConfigurationServer saved \"" + hostIpAdr + "\" as host.");
                  }
              }
  
              // generate a single sign-on key
              updateSSOKey();
  
              // Create default network offerings
              createDefaultNetworkOfferings();
  
              // Create default networks
              createDefaultNetworks();
  
              // Create userIpAddress ranges
  
              // Update existing vlans with networkId
              List<VlanVO> vlans = _vlanDao.listAll();
              if (vlans != null && !vlans.isEmpty()) {
                  for (final VlanVO vlan : vlans) {
                      if (vlan.getNetworkId().longValue() == 0) {
                          updateVlanWithNetworkId(vlan);
                      }
  
                      // Create vlan user_ip_address range
                      String ipPange = vlan.getIpRange();
                      String[] range = ipPange.split("-");
                      final String startIp = range[0];
                      final String endIp = range[1];
  
                      Transaction.execute(new TransactionCallbackNoReturn() {
                          @Override
                          public void doInTransactionWithoutResult(TransactionStatus status) {
                              IPRangeConfig config = new IPRangeConfig();
                              long startIPLong = NetUtils.ip2Long(startIp);
                              long endIPLong = NetUtils.ip2Long(endIp);
                              config.savePublicIPRange(TransactionLegacy.currentTxn(), startIPLong, endIPLong, vlan.getDataCenterId(), vlan.getId(), vlan.getNetworkId(),
                                      vlan.getPhysicalNetworkId(), false);
                          }
                      });
  
                  }
              }
          }
          // Update resource count if needed
          updateResourceCount();
  
          // store the public and private keys in the database
          updateKeyPairs();
  
          // generate a PSK to communicate with SSVM
          updateSecondaryStorageVMSharedKey();
  
          // generate a random password for system vm
          updateSystemvmPassword();
  
          // generate a random password used to authenticate zone-to-zone copy
          generateSecStorageVmCopyPassword();
  
          // Update the cloud identifier
          updateCloudIdentifier();
  
          _configDepotAdmin.populateConfigurations();
          // setup XenServer default PV driver version
          initiateXenServerPVDriverVersion();
  
          // We should not update seed data UUID column here since this will be invoked in upgrade case as well.
          //updateUuids();
          // Set init to true
          _configDao.update("init", "Hidden", "true");
  
          // invalidate cache in DAO as we have changed DB status
          _configDao.invalidateCache();
      }
  
      private void templateDetailsInitIfNotExist(long id, String name, String value) {
          TransactionLegacy txn = TransactionLegacy.currentTxn();
          PreparedStatement stmt = null;
          PreparedStatement stmtInsert = null;
          boolean insert = false;
          try {
              txn.start();
              stmt = txn.prepareAutoCloseStatement("SELECT id FROM vm_template_details WHERE template_id=? and name=?");
              stmt.setLong(1, id);
              stmt.setString(2, name);
              ResultSet rs = stmt.executeQuery();
              if (rs == null || !rs.next()) {
                  insert = true;
              }
              stmt.close();
  
              if (insert) {
                  stmtInsert = txn.prepareAutoCloseStatement("INSERT INTO vm_template_details(template_id, name, value) VALUES(?, ?, ?)");
                  stmtInsert.setLong(1, id);
                  stmtInsert.setString(2, name);
                  stmtInsert.setString(3, value);
                  if (stmtInsert.executeUpdate() < 1) {
                      throw new CloudRuntimeException("Unable to init template " + id + " datails: " + name);
                  }
              }
              txn.commit();
          } catch (Exception e) {
              logger.warn("Unable to init template " + id + " datails: " + name, e);
              throw new CloudRuntimeException("Unable to init template " + id + " datails: " + name);
          }
      }
  
      private void initiateXenServerPVDriverVersion() {
          Transaction.execute(new TransactionCallbackNoReturn() {
              @Override
              public void doInTransactionWithoutResult(TransactionStatus status) {
                  TransactionLegacy txn = TransactionLegacy.currentTxn();
                  String pvdriverversion = Config.XenServerPVdriverVersion.getDefaultValue();
                  PreparedStatement pstmt = null;
                  ResultSet rs1 = null;
                  ResultSet rs2 = null;
                  try {
                      String oldValue = _configDao.getValue(Config.XenServerPVdriverVersion.key());
                      if (oldValue == null) {
                          String sql = "select resource from host where hypervisor_type='XenServer' and removed is null and status not in ('Error', 'Removed') group by resource";
                          pstmt = txn.prepareAutoCloseStatement(sql);
                          rs1 = pstmt.executeQuery();
                          while (rs1.next()) {
                              String resource = rs1.getString(1); //resource column
                              if (resource == null)
                                  continue;
                              if (resource.equalsIgnoreCase("com.cloud.hypervisor.xenserver.resource.XenServer56Resource")
                                      || resource.equalsIgnoreCase("com.cloud.hypervisor.xenserver.resource.XenServer56FP1Resource")
                                      || resource.equalsIgnoreCase("com.cloud.hypervisor.xenserver.resource.XenServer56SP2Resource")
                                      || resource.equalsIgnoreCase("com.cloud.hypervisor.xenserver.resource.XenServer600Resource")
                                      || resource.equalsIgnoreCase("com.cloud.hypervisor.xenserver.resource.XenServer602Resource")) {
                                  pvdriverversion = "xenserver56";
                                  break;
                              }
                          }
                          _configDao.getValueAndInitIfNotExist(Config.XenServerPVdriverVersion.key(), Config.XenServerPVdriverVersion.getCategory(), pvdriverversion,
                                  Config.XenServerPVdriverVersion.getDescription());
                          sql = "select id from vm_template where hypervisor_type='XenServer'  and format!='ISO' and removed is null";
                          pstmt = txn.prepareAutoCloseStatement(sql);
                          rs2 = pstmt.executeQuery();
                          List<Long> tmpl_ids = new ArrayList<Long>();
                          while (rs2.next()) {
                              tmpl_ids.add(rs2.getLong(1));
                          }
                          for (Long tmpl_id : tmpl_ids) {
                              templateDetailsInitIfNotExist(tmpl_id, "hypervisortoolsversion", pvdriverversion);
                          }
                      }
                  } catch (Exception e) {
                      logger.debug("initiateXenServerPVDriverVersion failed due to " + e.toString());
                      // ignore
                  }
              }
          });
      }
  
      private String getMountParent() {
          return getEnvironmentProperty("mount.parent");
      }
  
      private String getEnvironmentProperty(String name) {
          try {
              final File propsFile = PropertiesUtil.findConfigFile("environment.properties");
  
              if (propsFile == null) {
                  return null;
              } else {
                  final Properties props = new Properties();
                  try(final FileInputStream finputstream = new FileInputStream(propsFile);) {
                      props.load(finputstream);
                  }catch (IOException e) {
                      logger.error("getEnvironmentProperty:Exception:" + e.getMessage());
                  }
                  return props.getProperty("mount.parent");
              }
          } catch (Exception e) {
              return null;
          }
      }
  
      @DB
      public void saveUser() {
          Transaction.execute(new TransactionCallbackNoReturn() {
              @Override
              public void doInTransactionWithoutResult(TransactionStatus status) {
                  TransactionLegacy txn = TransactionLegacy.currentTxn();
                  // insert system account
                  String insertSql = "INSERT INTO `cloud`.`account` (id, uuid, account_name, type, role_id, domain_id, account.default) VALUES (1, UUID(), 'system', '1', '1', '1', 1)";
  
                  try {
                      PreparedStatement stmt = txn.prepareAutoCloseStatement(insertSql);
                      stmt.executeUpdate();
                  } catch (SQLException ex) {
                      logger.debug("Looks like system account already exists");
                  }
                  // insert system user
                  insertSql = "INSERT INTO `cloud`.`user` (id, uuid, username, password, account_id, firstname, lastname, created, user.default)"
                          + " VALUES (1, UUID(), 'system', RAND(), 1, 'system', 'cloud', now(), 1)";
  
                  try {
                      PreparedStatement stmt = txn.prepareAutoCloseStatement(insertSql);
                      stmt.executeUpdate();
                  } catch (SQLException ex) {
                      logger.debug("Looks like system user already exists");
                  }
  
                  // insert admin user, but leave the account disabled until we set a
                  // password with the user authenticator
                  long id = 2;
                  String username = "admin";
                  String firstname = "admin";
                  String lastname = "cloud";
  
                  // create an account for the admin user first
                  insertSql = "INSERT INTO `cloud`.`account` (id, uuid, account_name, type, role_id, domain_id, account.default) VALUES (" + id + ", UUID(), '" + username
                          + "', '1', '1', '1', 1)";
                  try {
                      PreparedStatement stmt = txn.prepareAutoCloseStatement(insertSql);
                      stmt.executeUpdate();
                  } catch (SQLException ex) {
                      logger.debug("Looks like admin account already exists");
                  }
  
                  // now insert the user
                  insertSql = "INSERT INTO `cloud`.`user` (id, uuid, username, password, account_id, firstname, lastname, created, state, user.default) " + "VALUES (" + id
                          + ", UUID(), '" + username + "', RAND(), 2, '" + firstname + "','" + lastname + "',now(), 'disabled', 1)";
  
                  try {
                      PreparedStatement stmt = txn.prepareAutoCloseStatement(insertSql);
                      stmt.executeUpdate();
                  } catch (SQLException ex) {
                      logger.debug("Looks like admin user already exists");
                  }
  
                  try {
                      String tableName = "security_group";
                      try {
                          String checkSql = "SELECT * from network_group";
                          PreparedStatement stmt = txn.prepareAutoCloseStatement(checkSql);
                          stmt.executeQuery();
                          tableName = "network_group";
                      } catch (Exception ex) {
                          // Ignore in case of exception, table must not exist
                      }
  
                      insertSql = "SELECT * FROM " + tableName + " where account_id=2 and name='default'";
                      PreparedStatement stmt = txn.prepareAutoCloseStatement(insertSql);
                      ResultSet rs = stmt.executeQuery();
                      if (!rs.next()) {
                          // save default security group
                          if (tableName.equals("security_group")) {
                              insertSql = "INSERT INTO " + tableName + " (uuid, name, description, account_id, domain_id) "
                                      + "VALUES (UUID(), 'default', 'Default Security Group', 2, 1)";
                          } else {
                              insertSql = "INSERT INTO " + tableName + " (name, description, account_id, domain_id, account_name) "
                                      + "VALUES ('default', 'Default Security Group', 2, 1, 'admin')";
                          }
  
                          try {
                              stmt = txn.prepareAutoCloseStatement(insertSql);
                              stmt.executeUpdate();
                          } catch (SQLException ex) {
                              logger.warn("Failed to create default security group for default admin account due to ", ex);
                          }
                      }
                      rs.close();
                  } catch (Exception ex) {
                      logger.warn("Failed to create default security group for default admin account due to ", ex);
                  }
              }
          });
      }
  
      protected void updateCloudIdentifier() {
          // Creates and saves a UUID as the cloud identifier
          String currentCloudIdentifier = _configDao.getValue("cloud.identifier");
          if (currentCloudIdentifier == null || currentCloudIdentifier.isEmpty()) {
              String uuid = UUID.randomUUID().toString();
              _configDao.update(Config.CloudIdentifier.key(), Config.CloudIdentifier.getCategory(), uuid);
          }
      }
  
      @DB
      protected void updateSystemvmPassword() {
          String userid = System.getProperty("user.name");
          if (!userid.startsWith("cloud")) {
              return;
          }
  
          if (!Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) {
              return;
          }
  
          String already = _configDao.getValue("system.vm.password");
          if (already == null) {
              TransactionLegacy txn = TransactionLegacy.currentTxn();
              try {
                  String rpassword = _mgrService.generateRandomPassword();
                  String wSql = "INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) "
                  + "VALUES ('Secure','DEFAULT', 'management-server','system.vm.password', ?,'randmon password generated each management server starts for system vm')";
                  PreparedStatement stmt = txn.prepareAutoCloseStatement(wSql);
                  stmt.setString(1, DBEncryptionUtil.encrypt(rpassword));
                  stmt.executeUpdate();
                  logger.info("Updated systemvm password in database");
              } catch (SQLException e) {
                  logger.error("Cannot retrieve systemvm password", e);
              }
          }
  
      }
  
      @Override
      @DB
      public void updateKeyPairs() {
          // Grab the SSH key pair and insert it into the database, if it is not present
  
          String username = System.getProperty("user.name");
          Boolean devel = Boolean.valueOf(_configDao.getValue("developer"));
          if (!username.equalsIgnoreCase("cloud") && !devel) {
              logger.warn("Systemvm keypairs could not be set. Management server should be run as cloud user, or in development mode.");
              return;
          }
          String already = _configDao.getValue("ssh.privatekey");
          String homeDir = System.getProperty("user.home");
          if (homeDir == null) {
              throw new CloudRuntimeException("Cannot get home directory for account: " + username);
          }
  
          if (logger.isInfoEnabled()) {
              logger.info("Processing updateKeyPairs");
          }
  
          if (homeDir != null && homeDir.startsWith("~")) {
              logger.error("No home directory was detected for the user '" + username + "'. Please check the profile of this user.");
              throw new CloudRuntimeException("No home directory was detected for the user '" + username + "'. Please check the profile of this user.");
          }
  
          // Using non-default file names (id_rsa.cloud and id_rsa.cloud.pub) in developer mode. This is to prevent SSH keys overwritten for user running management server
          File privkeyfile = null;
          File pubkeyfile = null;
          if (devel) {
              privkeyfile = new File(homeDir + "/.ssh/id_rsa.cloud");
              pubkeyfile = new File(homeDir + "/.ssh/id_rsa.cloud.pub");
          } else {
              privkeyfile = new File(homeDir + "/.ssh/id_rsa");
              pubkeyfile = new File(homeDir + "/.ssh/id_rsa.pub");
          }
  
          if (already == null || already.isEmpty()) {
              if (logger.isInfoEnabled()) {
                  logger.info("Systemvm keypairs not found in database. Need to store them in the database");
              }
              // FIXME: take a global database lock here for safety.
              boolean onWindows = isOnWindows();
              if(!onWindows) {
                  Script.runSimpleBashScript("if [ -f " + privkeyfile + " ]; then rm -f " + privkeyfile + "; fi; ssh-keygen -t ecdsa -m PEM -N '' -f " + privkeyfile + " -q 2>/dev/null || ssh-keygen -t ecdsa -N '' -f " + privkeyfile + " -q");
              }
  
              final String privateKey;
              final String publicKey;
              try {
                  privateKey = new String(Files.readAllBytes(privkeyfile.toPath()));
              } catch (IOException e) {
                  logger.error("Cannot read the private key file", e);
                  throw new CloudRuntimeException("Cannot read the private key file");
              }
              try {
                  publicKey = new String(Files.readAllBytes(pubkeyfile.toPath()));
              } catch (IOException e) {
                  logger.error("Cannot read the public key file", e);
                  throw new CloudRuntimeException("Cannot read the public key file");
              }
  
              final String insertSql1 =
                      "INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) " +
                              "VALUES ('Hidden','DEFAULT', 'management-server','ssh.privatekey', '" + DBEncryptionUtil.encrypt(privateKey) +
                              "','Private key for the entire CloudStack')";
              final String insertSql2 =
                      "INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) " +
                              "VALUES ('Hidden','DEFAULT', 'management-server','ssh.publickey', '" + DBEncryptionUtil.encrypt(publicKey) +
                              "','Public key for the entire CloudStack')";
  
              Transaction.execute(new TransactionCallbackNoReturn() {
                  @Override
                  public void doInTransactionWithoutResult(TransactionStatus status) {
  
                      TransactionLegacy txn = TransactionLegacy.currentTxn();
                      try {
                          PreparedStatement stmt1 = txn.prepareAutoCloseStatement(insertSql1);
                          stmt1.executeUpdate();
                          if (logger.isDebugEnabled()) {
                              logger.debug("Private key inserted into database");
                          }
                      } catch (SQLException ex) {
                          logger.error("SQL of the private key failed", ex);
                          throw new CloudRuntimeException("SQL of the private key failed");
                      }
  
                      try {
                          PreparedStatement stmt2 = txn.prepareAutoCloseStatement(insertSql2);
                          stmt2.executeUpdate();
                          if (logger.isDebugEnabled()) {
                              logger.debug("Public key inserted into database");
                          }
                      } catch (SQLException ex) {
                          logger.error("SQL of the public key failed", ex);
                          throw new CloudRuntimeException("SQL of the public key failed");
                      }
                  }
              });
  
          } else {
              logger.info("Keypairs already in database, updating local copy");
              updateKeyPairsOnDisk(homeDir);
          }
          try {
              copyPrivateKeyToHosts(pubkeyfile.getAbsolutePath(), privkeyfile.getAbsolutePath());
          } catch (CloudRuntimeException e) {
              if (!devel) {
                  throw new CloudRuntimeException(e.getMessage());
              }
          }
      }
  
      @Override
      public List<ConfigurationVO> getConfigListByScope(String scope, Long resourceId) {
  
          // Getting the list of parameters defined at the scope
          Set<ConfigKey<?>> configList = _configDepot.getConfigListByScope(scope);
          List<ConfigurationVO> configVOList = new ArrayList<ConfigurationVO>();
          for (ConfigKey<?> param : configList) {
              ConfigurationVO configVo = _configDao.findByName(param.toString());
              configVo.setValue(_configDepot.get(param.toString()).valueIn(resourceId).toString());
              configVOList.add(configVo);
          }
          return configVOList;
      }
  
      private void writeKeyToDisk(String key, String keyPath) {
          File keyfile = new File(keyPath);
          if (!keyfile.exists()) {
              try {
                  keyfile.createNewFile();
              } catch (IOException e) {
                  logger.warn("Failed to create file: " + e.toString());
                  throw new CloudRuntimeException("Failed to update keypairs on disk: cannot create  key file " + keyPath);
              }
          }
  
          if (keyfile.exists()) {
              try (FileOutputStream kStream = new FileOutputStream(keyfile);){
                  if (kStream != null) {
                      kStream.write(key.getBytes());
                  }
              } catch (FileNotFoundException e) {
                  logger.warn("Failed to write  key to " + keyfile.getAbsolutePath(), e);
                  throw new CloudRuntimeException("Failed to update keypairs on disk: cannot find  key file " + keyPath);
              } catch (IOException e) {
                  logger.warn("Failed to write  key to " + keyfile.getAbsolutePath(), e);
                  throw new CloudRuntimeException("Failed to update keypairs on disk: cannot write to  key file " + keyPath);
              }
          }
  
      }
  
      private void updateKeyPairsOnDisk(String homeDir) {
          File keyDir = new File(homeDir + "/.ssh");
          Boolean devel = Boolean.valueOf(_configDao.getValue("developer"));
          if (!keyDir.isDirectory()) {
              logger.warn("Failed to create " + homeDir + "/.ssh for storing the SSH keypars");
              keyDir.mkdirs();
          }
          String pubKey = _configDao.getValue("ssh.publickey");
          String prvKey = _configDao.getValue("ssh.privatekey");
  
          // Using non-default file names (id_rsa.cloud and id_rsa.cloud.pub) in developer mode. This is to prevent SSH keys overwritten for user running management server
          if (devel) {
              writeKeyToDisk(prvKey, homeDir + "/.ssh/id_rsa.cloud");
              writeKeyToDisk(pubKey, homeDir + "/.ssh/id_rsa.cloud.pub");
          } else {
              writeKeyToDisk(prvKey, homeDir + "/.ssh/id_rsa");
              writeKeyToDisk(pubKey, homeDir + "/.ssh/id_rsa.pub");
          }
      }
  
      protected void copyPrivateKeyToHosts(String publicKeyPath, String privKeyPath) {
          logger.info("Trying to copy private keys to hosts");
          String injectScript = getInjectScript();
          String scriptPath = Script.findScript("", injectScript);
          if (scriptPath == null) {
              throw new CloudRuntimeException("Unable to find key inject script " + injectScript);
          }
  
          Script command = null;
          if(isOnWindows()) {
              command = new Script("python", logger);
          } else {
              command = new Script("/bin/bash", logger);
          }
          if (isOnWindows()) {
              scriptPath = scriptPath.replaceAll("\\\\" ,"/" );
              privKeyPath = privKeyPath.replaceAll("\\\\" ,"/" );
          }
  
          command.add(scriptPath);
          command.add(privKeyPath);
          final String result = command.execute();
          logger.info("The script injectkeys.sh was run with result : " + result);
          if (result != null) {
              logger.warn("The script injectkeys.sh failed to run successfully : " + result);
              throw new CloudRuntimeException("The script injectkeys.sh failed to run successfully : " + result);
          }
      }
  
      protected String getInjectScript() {
          String injectScript = null;
          boolean onWindows = isOnWindows();
          if(onWindows) {
              injectScript = "scripts/vm/systemvm/injectkeys.py";
          } else {
              injectScript = "scripts/vm/systemvm/injectkeys.sh";
          }
          return injectScript;
      }
  
      protected boolean isOnWindows() {
          String os = System.getProperty("os.name", "generic").toLowerCase();
          boolean onWindows = (os != null && os.startsWith("windows"));
          return onWindows;
      }
  
      @DB
      protected void generateSecStorageVmCopyPassword() {
          String already = _configDao.getValue("secstorage.copy.password");
  
          if (already == null) {
  
              logger.info("Need to store secondary storage vm copy password in the database");
              String password = PasswordGenerator.generateRandomPassword(12);
  
              final String insertSql1 =
                      "INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) " +
                              "VALUES ('Hidden','DEFAULT', 'management-server','secstorage.copy.password', '" + DBEncryptionUtil.encrypt(password) +
                              "','Password used to authenticate zone-to-zone template copy requests')";
              Transaction.execute(new TransactionCallbackNoReturn() {
                  @Override
                  public void doInTransactionWithoutResult(TransactionStatus status) {
  
                      TransactionLegacy txn = TransactionLegacy.currentTxn();
                      try {
                          PreparedStatement stmt1 = txn.prepareAutoCloseStatement(insertSql1);
                          stmt1.executeUpdate();
                          logger.debug("secondary storage vm copy password inserted into database");
                      } catch (SQLException ex) {
                          logger.warn("Failed to insert secondary storage vm copy password", ex);
                      }
                  }
              });
          }
      }
  
      private void updateSSOKey() {
          try {
              _configDao.update(Config.SSOKey.key(), Config.SSOKey.getCategory(), getPrivateKey());
          } catch (NoSuchAlgorithmException ex) {
              logger.error("error generating sso key", ex);
          }
      }
  
      /**
       * preshared key to be used by management server to communicate with SSVM during volume/template upload
       */
      private void updateSecondaryStorageVMSharedKey() {
          try {
              ConfigurationVO configInDB = _configDao.findByName(Config.SSVMPSK.key());
              if(configInDB == null) {
                  ConfigurationVO configVO = new ConfigurationVO(Config.SSVMPSK.getCategory(), "DEFAULT", Config.SSVMPSK.getComponent(), Config.SSVMPSK.key(), getPrivateKey(),
                          Config.SSVMPSK.getDescription());
                  logger.info("generating a new SSVM PSK. This goes to SSVM on Start");
                  _configDao.persist(configVO);
              } else if (StringUtils.isEmpty(configInDB.getValue())) {
                  logger.info("updating the SSVM PSK with new value. This goes to SSVM on Start");
                  _configDao.update(Config.SSVMPSK.key(), Config.SSVMPSK.getCategory(), getPrivateKey());
              }
          } catch (NoSuchAlgorithmException ex) {
              logger.error("error generating ssvm psk", ex);
          }
      }
  
      private String getPrivateKey() throws NoSuchAlgorithmException {
          String encodedKey = null;
          // Algorithm for generating Key is SHA1, should this be configurable?
          KeyGenerator generator = KeyGenerator.getInstance("HmacSHA1");
          SecretKey key = generator.generateKey();
          encodedKey = Base64.encodeBase64URLSafeString(key.getEncoded());
          return encodedKey;
  
      }
  
  
      @DB
      protected HostPodVO createPod(long userId, String podName, final long zoneId, String gateway, String cidr, final String startIp, String endIp)
              throws InternalErrorException {
          String[] cidrPair = cidr.split("\\/");
          String cidrAddress = cidrPair[0];
          int cidrSize = Integer.parseInt(cidrPair[1]);
  
          if (startIp != null) {
              if (endIp == null) {
                  endIp = NetUtils.getIpRangeEndIpFromCidr(cidrAddress, cidrSize);
              }
          }
  
          // Create the new pod in the database
          String ipRange;
          if (startIp != null) {
              ipRange = startIp + "-";
              if (endIp != null) {
                  ipRange += endIp;
              }
          } else {
              ipRange = "";
          }
  
          final HostPodVO pod = new HostPodVO(podName, zoneId, gateway, cidrAddress, cidrSize, ipRange);
          try {
              final String endIpFinal = endIp;
              Transaction.execute(new TransactionCallbackWithExceptionNoReturn<InternalErrorException>() {
                  @Override
                  public void doInTransactionWithoutResult(TransactionStatus status) throws InternalErrorException {
                      if (_podDao.persist(pod) == null) {
                          throw new InternalErrorException("Failed to create new pod. Please contact Cloud Support.");
                      }
  
                      if (startIp != null) {
                          _zoneDao.addPrivateIpAddress(zoneId, pod.getId(), startIp, endIpFinal, false, null);
                      }
  
                      String ipNums = _configDao.getValue("linkLocalIp.nums");
                      int nums = Integer.parseInt(ipNums);
                      if (nums > 16 || nums <= 0) {
                          throw new InvalidParameterValueException("The linkLocalIp.nums: " + nums + "is wrong, should be 1~16");
                      }
                      /* local link ip address starts from 169.254.0.2 - 169.254.(nums) */
                      String[] linkLocalIpRanges = NetUtils.getLinkLocalIPRange(_configDao.getValue(Config.ControlCidr.key()));
                      _zoneDao.addLinkLocalIpAddress(zoneId, pod.getId(), linkLocalIpRanges[0], linkLocalIpRanges[1]);
                  }
              });
          } catch (Exception e) {
              logger.error("Unable to create new pod due to " + e.getMessage(), e);
              throw new InternalErrorException("Failed to create new pod. Please contact Cloud Support.");
          }
  
          return pod;
      }
  
      private DiskOfferingVO createDefaultDiskOffering(String name, String description, ProvisioningType provisioningType,
                                                       int numGibibytes, String tags, boolean isCustomized, boolean isSystemUse) {
          long diskSize = numGibibytes;
          diskSize = diskSize * 1024 * 1024 * 1024;
          tags = cleanupTags(tags);
  
          DiskOfferingVO newDiskOffering = new DiskOfferingVO(name, description, provisioningType, diskSize, tags, isCustomized, null, null, null);
          newDiskOffering.setUniqueName("Cloud.Com-" + name);
          // leaving the above reference to cloud.com in as it is an identifier and has no real world relevance
          newDiskOffering = _diskOfferingDao.persistDefaultDiskOffering(newDiskOffering);
          return newDiskOffering;
      }
  
      private ServiceOfferingVO createServiceOffering(long userId, String name, int cpu, int ramSize, int speed, String displayText,
              ProvisioningType provisioningType, boolean localStorageRequired, boolean offerHA, String tags) {
          tags = cleanupTags(tags);
          DiskOfferingVO diskOfferingVO = new DiskOfferingVO(name, displayText, provisioningType, false, tags, false, false, true);
          diskOfferingVO.setUniqueName("Cloud.Com-" + name);
          diskOfferingVO = _diskOfferingDao.persistDefaultDiskOffering(diskOfferingVO);
  
          ServiceOfferingVO offering =
                  new ServiceOfferingVO(name, cpu, ramSize, speed, null, null, offerHA, displayText, false, null, false);
          offering.setUniqueName("Cloud.Com-" + name);
          offering.setDiskOfferingId(diskOfferingVO.getId());
          // leaving the above reference to cloud.com in as it is an identifyer and has no real world relevance
          offering = _serviceOfferingDao.persistSystemServiceOffering(offering);
          return offering;
      }
  
      private String cleanupTags(String tags) {
          if (tags != null) {
              String[] tokens = tags.split(",");
              StringBuilder t = new StringBuilder();
              for (int i = 0; i < tokens.length; i++) {
                  t.append(tokens[i].trim()).append(",");
              }
              t.delete(t.length() - 1, t.length());
              tags = t.toString();
          }
  
          return tags;
      }
  
      @DB
      protected void createDefaultNetworkOfferings() {
  
          NetworkOfferingVO publicNetworkOffering = new NetworkOfferingVO(NetworkOffering.SystemPublicNetwork, TrafficType.Public, true);
          publicNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(publicNetworkOffering);
          NetworkOfferingVO managementNetworkOffering = new NetworkOfferingVO(NetworkOffering.SystemManagementNetwork, TrafficType.Management, false);
          managementNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(managementNetworkOffering);
          NetworkOfferingVO controlNetworkOffering = new NetworkOfferingVO(NetworkOffering.SystemControlNetwork, TrafficType.Control, false);
          controlNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(controlNetworkOffering);
          NetworkOfferingVO storageNetworkOffering = new NetworkOfferingVO(NetworkOffering.SystemStorageNetwork, TrafficType.Storage, true);
          storageNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(storageNetworkOffering);
          NetworkOfferingVO privateGatewayNetworkOffering = new NetworkOfferingVO(NetworkOffering.SystemPrivateGatewayNetworkOffering, GuestType.Isolated, true);
          privateGatewayNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(privateGatewayNetworkOffering);
          NetworkOfferingVO privateGatewayNetworkOfferingWithoutVlan = new NetworkOfferingVO(NetworkOffering.SystemPrivateGatewayNetworkOfferingWithoutVlan, GuestType.Isolated, false);
          privateGatewayNetworkOfferingWithoutVlan = _networkOfferingDao.persistDefaultNetworkOffering(privateGatewayNetworkOfferingWithoutVlan);
  
          //populate providers
          final Map<Network.Service, Network.Provider> defaultSharedNetworkOfferingProviders = new HashMap<Network.Service, Network.Provider>();
          defaultSharedNetworkOfferingProviders.put(Service.Dhcp, Provider.VirtualRouter);
          defaultSharedNetworkOfferingProviders.put(Service.Dns, Provider.VirtualRouter);
          defaultSharedNetworkOfferingProviders.put(Service.UserData, Provider.VirtualRouter);
  
          final Map<Network.Service, Network.Provider> defaultIsolatedNetworkOfferingProviders = defaultSharedNetworkOfferingProviders;
  
          final Map<Network.Service, Network.Provider> defaultSharedSGNetworkOfferingProviders = new HashMap<Network.Service, Network.Provider>();
          defaultSharedSGNetworkOfferingProviders.put(Service.Dhcp, Provider.VirtualRouter);
          defaultSharedSGNetworkOfferingProviders.put(Service.Dns, Provider.VirtualRouter);
          defaultSharedSGNetworkOfferingProviders.put(Service.UserData, Provider.VirtualRouter);
          defaultSharedSGNetworkOfferingProviders.put(Service.SecurityGroup, Provider.SecurityGroupProvider);
  
          final Map<Network.Service, Network.Provider> defaultTungstenSharedSGNetworkOfferingProviders = new HashMap<>();
          defaultTungstenSharedSGNetworkOfferingProviders.put(Service.Connectivity, Provider.Tungsten);
          defaultTungstenSharedSGNetworkOfferingProviders.put(Service.Dhcp, Provider.Tungsten);
          defaultTungstenSharedSGNetworkOfferingProviders.put(Service.Dns, Provider.Tungsten);
          defaultTungstenSharedSGNetworkOfferingProviders.put(Service.SecurityGroup, Provider.Tungsten);
  
          final Map<Network.Service, Network.Provider> defaultIsolatedSourceNatEnabledNetworkOfferingProviders = new HashMap<Network.Service, Network.Provider>();
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.Dhcp, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.Dns, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.UserData, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.Firewall, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.Gateway, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.Lb, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.SourceNat, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.StaticNat, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.PortForwarding, Provider.VirtualRouter);
          defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.Vpn, Provider.VirtualRouter);
  
          final Map<Network.Service, Network.Provider> netscalerServiceProviders = new HashMap<Network.Service, Network.Provider>();
          netscalerServiceProviders.put(Service.Dhcp, Provider.VirtualRouter);
          netscalerServiceProviders.put(Service.Dns, Provider.VirtualRouter);
          netscalerServiceProviders.put(Service.UserData, Provider.VirtualRouter);
          netscalerServiceProviders.put(Service.SecurityGroup, Provider.SecurityGroupProvider);
          netscalerServiceProviders.put(Service.StaticNat, Provider.Netscaler);
          netscalerServiceProviders.put(Service.Lb, Provider.Netscaler);
  
          // The only one diff between 1 and 2 network offerings is that the first one has SG enabled. In Basic zone only
          // first network offering has to be enabled, in Advance zone - the second one
          Transaction.execute(new TransactionCallbackNoReturn() {
              @Override
              public void doInTransactionWithoutResult(TransactionStatus status) {
                  // Offering #1
                  NetworkOfferingVO defaultSharedSGNetworkOffering =
                          new NetworkOfferingVO(NetworkOffering.DefaultSharedNetworkOfferingWithSGService, "Offering for Shared Security group enabled networks",
                                  TrafficType.Guest, false, true, null, null, true, Availability.Optional, null, Network.GuestType.Shared, true, true, false, false, false, false);
  
                  defaultSharedSGNetworkOffering.setState(NetworkOffering.State.Enabled);
                  defaultSharedSGNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(defaultSharedSGNetworkOffering);
  
                  for (Service service : defaultSharedSGNetworkOfferingProviders.keySet()) {
                      NetworkOfferingServiceMapVO offService =
                              new NetworkOfferingServiceMapVO(defaultSharedSGNetworkOffering.getId(), service, defaultSharedSGNetworkOfferingProviders.get(service));
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  // Offering #2
                  NetworkOfferingVO defaultSharedNetworkOffering =
                          new NetworkOfferingVO(NetworkOffering.DefaultSharedNetworkOffering, "Offering for Shared networks", TrafficType.Guest, false, true, null, null, true,
                                  Availability.Optional, null, Network.GuestType.Shared, true, true, false, false, false, false);
  
                  defaultSharedNetworkOffering.setState(NetworkOffering.State.Enabled);
                  defaultSharedNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(defaultSharedNetworkOffering);
  
                  for (Service service : defaultSharedNetworkOfferingProviders.keySet()) {
                      NetworkOfferingServiceMapVO offService =
                              new NetworkOfferingServiceMapVO(defaultSharedNetworkOffering.getId(), service, defaultSharedNetworkOfferingProviders.get(service));
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  NetworkOfferingVO defaultTungstenSharedSGNetworkOffering =
                          new NetworkOfferingVO(NetworkOffering.DEFAULT_TUNGSTEN_SHARED_NETWORK_OFFERING_WITH_SGSERVICE, "Offering for Tungsten Shared Security group enabled networks",
                                  TrafficType.Guest, false, true, null, null, true, Availability.Optional, null, Network.GuestType.Shared, true, true, false, false, false, false);
                  defaultTungstenSharedSGNetworkOffering.setState(NetworkOffering.State.Enabled);
                  defaultTungstenSharedSGNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(defaultTungstenSharedSGNetworkOffering);
  
                  for (Map.Entry<Network.Service, Network.Provider> service : defaultTungstenSharedSGNetworkOfferingProviders.entrySet()) {
                      NetworkOfferingServiceMapVO offService =
                              new NetworkOfferingServiceMapVO(defaultTungstenSharedSGNetworkOffering.getId(), service.getKey(), service.getValue());
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  // Offering #3
                  NetworkOfferingVO defaultIsolatedSourceNatEnabledNetworkOffering =
                          new NetworkOfferingVO(NetworkOffering.DefaultIsolatedNetworkOfferingWithSourceNatService,
                                  "Offering for Isolated networks with Source Nat service enabled", TrafficType.Guest, false, false, null, null, true, Availability.Required, null,
                                  Network.GuestType.Isolated, true, false, false, false, true, false);
  
                  defaultIsolatedSourceNatEnabledNetworkOffering.setState(NetworkOffering.State.Enabled);
                  defaultIsolatedSourceNatEnabledNetworkOffering.setSupportsVmAutoScaling(true);
                  defaultIsolatedSourceNatEnabledNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(defaultIsolatedSourceNatEnabledNetworkOffering);
  
                  for (Service service : defaultIsolatedSourceNatEnabledNetworkOfferingProviders.keySet()) {
                      NetworkOfferingServiceMapVO offService =
                              new NetworkOfferingServiceMapVO(defaultIsolatedSourceNatEnabledNetworkOffering.getId(), service,
                                      defaultIsolatedSourceNatEnabledNetworkOfferingProviders.get(service));
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  // Offering #4
                  NetworkOfferingVO defaultIsolatedEnabledNetworkOffering =
                          new NetworkOfferingVO(NetworkOffering.DefaultIsolatedNetworkOffering, "Offering for Isolated networks with no Source Nat service", TrafficType.Guest,
                                  false, true, null, null, true, Availability.Optional, null, Network.GuestType.Isolated, true, true, false, false, false, false);
  
                  defaultIsolatedEnabledNetworkOffering.setState(NetworkOffering.State.Enabled);
                  defaultIsolatedEnabledNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(defaultIsolatedEnabledNetworkOffering);
  
                  for (Service service : defaultIsolatedNetworkOfferingProviders.keySet()) {
                      NetworkOfferingServiceMapVO offService =
                              new NetworkOfferingServiceMapVO(defaultIsolatedEnabledNetworkOffering.getId(), service, defaultIsolatedNetworkOfferingProviders.get(service));
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  // Offering #5
                  NetworkOfferingVO defaultNetscalerNetworkOffering =
                          new NetworkOfferingVO(NetworkOffering.DefaultSharedEIPandELBNetworkOffering,
                                  "Offering for Shared networks with Elastic IP and Elastic LB capabilities", TrafficType.Guest, false, true, null, null, true,
                                  Availability.Optional, null, Network.GuestType.Shared, true, false, false, false, true, true, true, false, false, true, true, false, false, false, false, false);
  
                  defaultNetscalerNetworkOffering.setState(NetworkOffering.State.Enabled);
                  defaultNetscalerNetworkOffering.setSupportsVmAutoScaling(true);
                  defaultNetscalerNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(defaultNetscalerNetworkOffering);
  
                  for (Service service : netscalerServiceProviders.keySet()) {
                      NetworkOfferingServiceMapVO offService =
                              new NetworkOfferingServiceMapVO(defaultNetscalerNetworkOffering.getId(), service, netscalerServiceProviders.get(service));
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  // Offering #6
                  NetworkOfferingVO defaultNetworkOfferingForVpcNetworks =
                          new NetworkOfferingVO(NetworkOffering.DefaultIsolatedNetworkOfferingForVpcNetworks,
                                  "Offering for Isolated Vpc networks with Source Nat service enabled", TrafficType.Guest, false, false, null, null, true, Availability.Optional,
                                  null, Network.GuestType.Isolated, true, false, false, false, true, true);
  
                  defaultNetworkOfferingForVpcNetworks.setState(NetworkOffering.State.Enabled);
                  defaultNetworkOfferingForVpcNetworks.setSupportsVmAutoScaling(true);
                  defaultNetworkOfferingForVpcNetworks = _networkOfferingDao.persistDefaultNetworkOffering(defaultNetworkOfferingForVpcNetworks);
  
                  Map<Network.Service, Network.Provider> defaultVpcNetworkOfferingProviders = new HashMap<Network.Service, Network.Provider>();
                  defaultVpcNetworkOfferingProviders.put(Service.Dhcp, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.Dns, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.UserData, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.NetworkACL, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.Gateway, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.Lb, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.SourceNat, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.StaticNat, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.PortForwarding, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProviders.put(Service.Vpn, Provider.VPCVirtualRouter);
  
                  for (Map.Entry<Service,Provider> entry : defaultVpcNetworkOfferingProviders.entrySet()) {
                       NetworkOfferingServiceMapVO offService =
                              new NetworkOfferingServiceMapVO(defaultNetworkOfferingForVpcNetworks.getId(), entry.getKey(), entry.getValue());
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  // Offering #7
                  NetworkOfferingVO defaultNetworkOfferingForVpcNetworksNoLB =
                          new NetworkOfferingVO(NetworkOffering.DefaultIsolatedNetworkOfferingForVpcNetworksNoLB,
                                  "Offering for Isolated Vpc networks with Source Nat service enabled and LB service Disabled", TrafficType.Guest, false, false, null, null, true,
                                  Availability.Optional, null, Network.GuestType.Isolated, false, false, false, false, false, true);
  
                  defaultNetworkOfferingForVpcNetworksNoLB.setState(NetworkOffering.State.Enabled);
                  defaultNetworkOfferingForVpcNetworksNoLB = _networkOfferingDao.persistDefaultNetworkOffering(defaultNetworkOfferingForVpcNetworksNoLB);
  
                  Map<Network.Service, Network.Provider> defaultVpcNetworkOfferingProvidersNoLB = new HashMap<Network.Service, Network.Provider>();
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.Dhcp, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.Dns, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.UserData, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.NetworkACL, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.Gateway, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.SourceNat, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.StaticNat, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.PortForwarding, Provider.VPCVirtualRouter);
                  defaultVpcNetworkOfferingProvidersNoLB.put(Service.Vpn, Provider.VPCVirtualRouter);
  
                  for (Map.Entry<Service,Provider> entry : defaultVpcNetworkOfferingProvidersNoLB.entrySet()) {
                      NetworkOfferingServiceMapVO offService =
                              new NetworkOfferingServiceMapVO(defaultNetworkOfferingForVpcNetworksNoLB.getId(), entry.getKey(), entry.getValue());
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  //offering #8 - network offering with internal lb service
                  NetworkOfferingVO internalLbOff =
                          new NetworkOfferingVO(NetworkOffering.DefaultIsolatedNetworkOfferingForVpcNetworksWithInternalLB,
                                  "Offering for Isolated Vpc networks with Internal LB support", TrafficType.Guest, false, false, null, null, true, Availability.Optional, null,
                                  Network.GuestType.Isolated, false, false, false, true, false, true);
  
                  internalLbOff.setState(NetworkOffering.State.Enabled);
                  internalLbOff = _networkOfferingDao.persistDefaultNetworkOffering(internalLbOff);
  
                  Map<Network.Service, Network.Provider> internalLbOffProviders = new HashMap<Network.Service, Network.Provider>();
                  internalLbOffProviders.put(Service.Dhcp, Provider.VPCVirtualRouter);
                  internalLbOffProviders.put(Service.Dns, Provider.VPCVirtualRouter);
                  internalLbOffProviders.put(Service.UserData, Provider.VPCVirtualRouter);
                  internalLbOffProviders.put(Service.NetworkACL, Provider.VPCVirtualRouter);
                  internalLbOffProviders.put(Service.Gateway, Provider.VPCVirtualRouter);
                  internalLbOffProviders.put(Service.Lb, Provider.InternalLbVm);
                  internalLbOffProviders.put(Service.SourceNat, Provider.VPCVirtualRouter);
  
                  for (Service service : internalLbOffProviders.keySet()) {
                      NetworkOfferingServiceMapVO offService = new NetworkOfferingServiceMapVO(internalLbOff.getId(), service, internalLbOffProviders.get(service));
                      _ntwkOfferingServiceMapDao.persist(offService);
                      logger.trace("Added service for the network offering: " + offService);
                  }
  
                  _networkOfferingDao.persistDefaultL2NetworkOfferings();
  
                  // Offering #9 - network offering for NSX provider - NATTED mode
                  createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING, "Offering for NSX enabled networks - NAT mode",
                          NetworkOffering.NetworkMode.NATTED, false, true, false, Provider.Nsx);
  
                  // Offering #10 - network offering for NSX provider - ROUTED mode
                  createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_ROUTED_NSX_OFFERING, "Offering for NSX enabled networks - ROUTED mode",
                          NetworkOffering.NetworkMode.ROUTED, false, true, false, Provider.Nsx);
  
                  // Offering #11 - network offering for NSX provider for VPCs - NATTED mode
                  createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING_FOR_VPC, "Offering for NSX enabled networks on VPCs - NAT mode",
                          NetworkOffering.NetworkMode.NATTED, true, true, false, Provider.Nsx);
  
                  // Offering #12 - network offering for NSX provider for VPCs - ROUTED mode
                  createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_ROUTED_NSX_OFFERING_FOR_VPC, "Offering for NSX enabled networks on VPCs - ROUTED mode",
                          NetworkOffering.NetworkMode.ROUTED, true, true, false, Provider.Nsx);
  
                  // Offering #13 - network offering for NSX provider for VPCs with Internal LB - NATTED mode
                  createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING_FOR_VPC_WITH_ILB, "Offering for NSX enabled networks on VPCs with internal LB - NAT mode",
                          NetworkOffering.NetworkMode.NATTED, true, false, false, Provider.Nsx);
  
                  // Offering #14 - network offering for Netris provider for VPCs - ROUTED mode
                  createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_ROUTED_NETRIS_OFFERING_FOR_VPC, "Offering for Netris enabled networks on VPCs - ROUTED mode",
                          NetworkOffering.NetworkMode.ROUTED, true, true, false, Provider.Netris);
  
                  // Offering #15 - network offering for Netris provider for VPCs - NATTED mode
                  createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_NAT_NETRIS_OFFERING_FOR_VPC, "Offering for Netris enabled networks on VPCs - NAT mode",
                          NetworkOffering.NetworkMode.NATTED, true, true, true, Provider.Netris);
              }
          });
      }
  
      private void createAndPersistDefaultProviderOffering(String name, String displayText, NetworkOffering.NetworkMode networkMode,
                                                      boolean forVpc, boolean publicLB, boolean supportVmAutoscaling, Provider provider) {
          NetworkOfferingVO providerNetworkOffering =
                  new NetworkOfferingVO(name, displayText, TrafficType.Guest, false, false, null,
                          null, true, Availability.Optional, null, GuestType.Isolated, false,
                          false, false, false, false, forVpc);
          providerNetworkOffering.setPublicLb(publicLB);
          providerNetworkOffering.setInternalLb(!publicLB);
          providerNetworkOffering.setNetworkMode(networkMode);
          providerNetworkOffering.setState(NetworkOffering.State.Enabled);
          providerNetworkOffering.setSupportsVmAutoScaling(supportVmAutoscaling);
          providerNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(providerNetworkOffering);
  
          Map<Service, Provider> serviceProviderMap = getServicesAndProvidersForProviderNetwork(networkMode, forVpc, provider);
          for (Map.Entry<Network.Service, Network.Provider> service : serviceProviderMap.entrySet()) {
              NetworkOfferingServiceMapVO offService =
                      new NetworkOfferingServiceMapVO(providerNetworkOffering.getId(), service.getKey(), service.getValue());
              _ntwkOfferingServiceMapDao.persist(offService);
              logger.trace("Added service for the network offering: " + offService);
          }
      }
  
      private Map<Service, Provider> getServicesAndProvidersForProviderNetwork(NetworkOffering.NetworkMode networkMode, boolean forVpc, Provider provider) {
          final Map<Network.Service, Network.Provider> serviceProviderMap = new HashMap<>();
          Provider routerProvider = forVpc ? Provider.VPCVirtualRouter : Provider.VirtualRouter;
          serviceProviderMap.put(Service.Dhcp, routerProvider);
          serviceProviderMap.put(Service.Dns, routerProvider);
          serviceProviderMap.put(Service.UserData, routerProvider);
          if (forVpc) {
              serviceProviderMap.put(Service.NetworkACL, provider);
          } else {
              serviceProviderMap.put(Service.Firewall, provider);
          }
          if (networkMode == NetworkOffering.NetworkMode.NATTED) {
              serviceProviderMap.put(Service.SourceNat, provider);
              serviceProviderMap.put(Service.StaticNat, provider);
              serviceProviderMap.put(Service.PortForwarding, provider);
              serviceProviderMap.put(Service.Lb, provider);
              if (provider == Provider.Netris) {
                  serviceProviderMap.put(Service.Vpn, routerProvider);
              }
          }
          return serviceProviderMap;
      }
  
      private void createDefaultNetworks() {
          List<DataCenterVO> zones = _dataCenterDao.listAll();
          long id = 1;
  
          HashMap<TrafficType, String> guruNames = new HashMap<TrafficType, String>();
          guruNames.put(TrafficType.Public, PublicNetworkGuru.class.getSimpleName());
          guruNames.put(TrafficType.Management, PodBasedNetworkGuru.class.getSimpleName());
          guruNames.put(TrafficType.Control, ControlNetworkGuru.class.getSimpleName());
          guruNames.put(TrafficType.Storage, StorageNetworkGuru.class.getSimpleName());
          guruNames.put(TrafficType.Guest, DirectPodBasedNetworkGuru.class.getSimpleName());
  
          for (DataCenterVO zone : zones) {
              long zoneId = zone.getId();
              long accountId = 1L;
              Long domainId = zone.getDomainId();
  
              if (domainId == null) {
                  domainId = 1L;
              }
              // Create default networks - system only
              List<NetworkOfferingVO> ntwkOff = _networkOfferingDao.listSystemNetworkOfferings();
  
              for (NetworkOfferingVO offering : ntwkOff) {
                  if (offering.isSystemOnly()) {
                      long related = id;
                      long networkOfferingId = offering.getId();
                      Mode mode = Mode.Static;
                      String networkDomain = null;
  
                      BroadcastDomainType broadcastDomainType = null;
                      TrafficType trafficType = offering.getTrafficType();
  
                      boolean specifyIpRanges = false;
  
                      if (trafficType == TrafficType.Management) {
                          broadcastDomainType = BroadcastDomainType.Native;
                      } else if (trafficType == TrafficType.Storage) {
                          broadcastDomainType = BroadcastDomainType.Native;
                          specifyIpRanges = true;
                      } else if (trafficType == TrafficType.Control) {
                          broadcastDomainType = BroadcastDomainType.LinkLocal;
                      } else if (offering.getTrafficType() == TrafficType.Public) {
                          if ((zone.getNetworkType() == NetworkType.Advanced && !zone.isSecurityGroupEnabled()) || zone.getNetworkType() == NetworkType.Basic) {
                              specifyIpRanges = true;
                              broadcastDomainType = BroadcastDomainType.Vlan;
                          } else {
                              continue;
                          }
                      }
  
                      if (broadcastDomainType != null) {
                          NetworkVO network =
                                  new NetworkVO(id, trafficType, mode, broadcastDomainType, networkOfferingId, domainId, accountId, related, null, null, networkDomain,
                                          Network.GuestType.Shared, zoneId, null, null, specifyIpRanges, null, offering.isRedundantRouter());
                          network.setGuruName(guruNames.get(network.getTrafficType()));
                          network.setDns1(zone.getDns1());
                          network.setDns2(zone.getDns2());
                          network.setState(State.Implemented);
                          _networkDao.persist(network, false, getServicesAndProvidersForNetwork(networkOfferingId));
                          id++;
                      }
                  }
              }
          }
      }
  
      private void updateVlanWithNetworkId(VlanVO vlan) {
          long zoneId = vlan.getDataCenterId();
          long networkId = 0L;
          DataCenterVO zone = _zoneDao.findById(zoneId);
  
          if (zone.getNetworkType() == NetworkType.Advanced) {
              networkId = getSystemNetworkIdByZoneAndTrafficType(zoneId, TrafficType.Public);
          } else {
              networkId = getSystemNetworkIdByZoneAndTrafficType(zoneId, TrafficType.Guest);
          }
  
          vlan.setNetworkId(networkId);
          _vlanDao.update(vlan.getId(), vlan);
      }
  
      private long getSystemNetworkIdByZoneAndTrafficType(long zoneId, TrafficType trafficType) {
          // find system public network offering
          Long networkOfferingId = null;
          List<NetworkOfferingVO> offerings = _networkOfferingDao.listSystemNetworkOfferings();
          for (NetworkOfferingVO offering : offerings) {
              if (offering.getTrafficType() == trafficType) {
                  networkOfferingId = offering.getId();
                  break;
              }
          }
  
          if (networkOfferingId == null) {
              throw new InvalidParameterValueException("Unable to find system network offering with traffic type " + trafficType);
          }
  
          List<NetworkVO> networks = _networkDao.listBy(Account.ACCOUNT_ID_SYSTEM, networkOfferingId, zoneId);
          if (networks == null || networks.isEmpty()) {
              throw new InvalidParameterValueException("Unable to find network with traffic type " + trafficType + " in zone " + zoneId);
          }
          return networks.get(0).getId();
      }
  
      @DB
      public void updateResourceCount() {
          ResourceType[] resourceTypes = Resource.ResourceType.values();
          List<AccountVO> accounts = _accountDao.listAll();
          List<DomainVO> domains = _domainDao.listAll();
          List<ResourceCountVO> domainResourceCount = _resourceCountDao.listResourceCountByOwnerType(ResourceOwnerType.Domain);
          List<ResourceCountVO> accountResourceCount = _resourceCountDao.listResourceCountByOwnerType(ResourceOwnerType.Account);
  
          final int expectedCount = resourceTypes.length;
  
          if ((domainResourceCount.size() < expectedCount * domains.size())) {
              logger.debug("resource_count table has records missing for some domains...going to insert them");
              for (final DomainVO domain : domains) {
                  // Lock domain
                  Transaction.execute(new TransactionCallbackNoReturn() {
                      @Override
                      public void doInTransactionWithoutResult(TransactionStatus status) {
                          _domainDao.lockRow(domain.getId(), true);
                          List<ResourceCountVO> domainCounts = _resourceCountDao.listByOwnerId(domain.getId(), ResourceOwnerType.Domain);
                          List<String> domainCountStr = new ArrayList<String>();
                          for (ResourceCountVO domainCount : domainCounts) {
                              domainCountStr.add(domainCount.getType().toString());
                          }
  
                          if (domainCountStr.size() < expectedCount) {
                              for (ResourceType resourceType : resourceTypes) {
                                  if (!domainCountStr.contains(resourceType.toString())) {
                                      ResourceCountVO resourceCountVO = new ResourceCountVO(resourceType, 0, domain.getId(), ResourceOwnerType.Domain);
                                      logger.debug("Inserting resource count of type {} for domain {}", resourceType, domain);
                                      _resourceCountDao.persist(resourceCountVO);
                                  }
                              }
                          }
                      }
                  });
  
              }
          }
  
          if ((accountResourceCount.size() < expectedCount * accounts.size())) {
              logger.debug("resource_count table has records missing for some accounts...going to insert them");
              for (final AccountVO account : accounts) {
                  // lock account
                  Transaction.execute(new TransactionCallbackNoReturn() {
                      @Override
                      public void doInTransactionWithoutResult(TransactionStatus status) {
                          _accountDao.lockRow(account.getId(), true);
                          List<ResourceCountVO> accountCounts = _resourceCountDao.listByOwnerId(account.getId(), ResourceOwnerType.Account);
                          List<String> accountCountStr = new ArrayList<String>();
                          for (ResourceCountVO accountCount : accountCounts) {
                              accountCountStr.add(accountCount.getType().toString());
                          }
  
                          if (accountCountStr.size() < expectedCount) {
                              for (ResourceType resourceType : resourceTypes) {
                                  if (!accountCountStr.contains(resourceType.toString())) {
                                      ResourceCountVO resourceCountVO = new ResourceCountVO(resourceType, 0, account.getId(), ResourceOwnerType.Account);
                                      logger.debug("Inserting resource count of type {} for account {}", resourceType, account);
                                      _resourceCountDao.persist(resourceCountVO);
                                  }
                              }
                          }
                      }
                  });
              }
          }
      }
  
      public Map<String, String> getServicesAndProvidersForNetwork(long networkOfferingId) {
          Map<String, String> svcProviders = new HashMap<String, String>();
          List<NetworkOfferingServiceMapVO> servicesMap = _ntwkOfferingServiceMapDao.listByNetworkOfferingId(networkOfferingId);
  
          for (NetworkOfferingServiceMapVO serviceMap : servicesMap) {
              if (svcProviders.containsKey(serviceMap.getService())) {
                  continue;
              }
              svcProviders.put(serviceMap.getService(), serviceMap.getProvider());
          }
  
          return svcProviders;
      }
  
  }
  