Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1324,6 +1324,7 @@ public class ApiConstants {
public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize";
public static final String SUPPORTS_HA = "supportsha";
public static final String SUPPORTS_AUTOSCALING = "supportsautoscaling";
public static final String SUPPORTS_ETCD = "supportsetcd";
public static final String AUTOSCALING_ENABLED = "autoscalingenabled";
public static final String MIN_SIZE = "minsize";
public static final String MAX_SIZE = "maxsize";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.cloudstack.storage.command;

import com.cloud.agent.api.Answer;

public class CheckFileInIsoAnswer extends Answer {

private boolean filePresent;

public CheckFileInIsoAnswer() {
super();
}

/**
* The ISO was read and the file was or was not found in it.
*/
public CheckFileInIsoAnswer(final CheckFileInIsoCommand cmd, final boolean filePresent) {
super(cmd, true, null);
this.filePresent = filePresent;
}

/**
* The ISO could not be read, so nothing is known about the file.
*/
public CheckFileInIsoAnswer(final CheckFileInIsoCommand cmd, final String errorDetails) {
super(cmd, false, errorDetails);
}

public boolean isFilePresent() {
return filePresent;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.cloudstack.storage.command;

import com.cloud.agent.api.to.DataStoreTO;

/**
* Asks the secondary storage VM whether an ISO on the given image store contains a file.
*/
public class CheckFileInIsoCommand extends StorageSubSystemCommand {

private DataStoreTO store;
private String isoPath;
private String filePath;

public CheckFileInIsoCommand() {
super();
}

public CheckFileInIsoCommand(final DataStoreTO store, final String isoPath, final String filePath) {
super();
this.store = store;
this.isoPath = isoPath;
this.filePath = filePath;
}

public DataStoreTO getStore() {
return store;
}

/**
* @return path of the ISO, relative to the root of the image store
*/
public String getIsoPath() {
return isoPath;
}

/**
* @return path of the file to look for, relative to the root of the ISO
*/
public String getFilePath() {
return filePath;
}

@Override
public boolean executeInSequence() {
return false;
}

@Override
public void setExecuteInSequence(boolean inSeq) {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@
import com.cloud.kubernetes.version.KubernetesSupportedVersion;
import com.cloud.kubernetes.version.KubernetesSupportedVersionVO;
import com.cloud.kubernetes.version.KubernetesVersionManagerImpl;
import com.cloud.kubernetes.version.KubernetesVersionService;
import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao;
import com.cloud.network.IpAddress;
import com.cloud.network.Network;
Expand Down Expand Up @@ -326,6 +327,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
@Inject
public KubernetesSupportedVersionDao kubernetesSupportedVersionDao;
@Inject
protected KubernetesVersionService kubernetesVersionService;
@Inject
protected SSHKeyPairDao sshKeyPairDao;
@Inject
protected DataCenterDao dataCenterDao;
Expand Down Expand Up @@ -1094,6 +1097,29 @@ public boolean isCommandSupported(KubernetesCluster cluster, String cmdName) {
}
}

/**
* Dedicated etcd nodes install etcd from the Kubernetes ISO, and not every ISO has it. Fail the request when
* the ISO is known not to have it. When the ISO could not be checked the request is let through, and a failed
* etcd setup is then caught while the cluster starts.
*/
protected void validateEtcdNodesSupportedByKubernetesVersion(final long etcdNodes, final KubernetesSupportedVersion kubernetesVersion, final long zoneId) {
if (etcdNodes <= 0) {
return;
}
Boolean etcdIncluded = kubernetesVersionService.isEtcdIncludedInIso(kubernetesVersion, zoneId);
if (etcdIncluded == null) {
logger.warn("Unable to tell whether the ISO of Kubernetes version {} includes the etcd binaries, allowing the request for {} etcd node(s)",
kubernetesVersion, etcdNodes);
return;
}
if (!etcdIncluded) {
throw new InvalidParameterValueException(String.format("Kubernetes version %s (ID: %s) cannot be used for a cluster with dedicated etcd nodes " +
"as its ISO does not include the etcd binaries (%s). Use a Kubernetes version whose ISO was built with etcd, " +
"or create the cluster without etcd nodes", kubernetesVersion.getName(), kubernetesVersion.getUuid(),
KubernetesVersionService.ETCD_ARCHIVE_PATH_IN_ISO));
}
}

private void validateManagedKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws CloudRuntimeException {
ApiServiceConfiguration.validateEndpointUrl();
final String name = cmd.getName();
Expand Down Expand Up @@ -1168,6 +1194,8 @@ private void validateManagedKubernetesClusterCreateParameters(final CreateKubern
throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state for datacenter ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid()));
}

validateEtcdNodesSupportedByKubernetesVersion(cmd.getEtcdNodes(), clusterKubernetesVersion, zone.getId());

validateServiceOfferingsForNodeTypes(serviceOfferingNodeTypeMap, defaultServiceOfferingId, cmd.getEtcdNodes(), clusterKubernetesVersion);

validateSshKeyPairForKubernetesCreateParameters(sshKeyPair, owner);
Expand Down Expand Up @@ -3089,7 +3117,8 @@ public ConfigKey<?>[] getConfigKeys() {
KubernetesControlNodeInstallReattempts,
KubernetesWorkerNodeInstallAttemptWait,
KubernetesWorkerNodeInstallReattempts,
KubernetesEtcdNodeStartPort
KubernetesEtcdNodeStartPort,
KubernetesEtcdNodeSetupTimeout
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,13 @@ public interface KubernetesClusterService extends PluggableService, Configurable
"Start port for Port forwarding rules for etcd nodes",
true,
KubernetesServiceEnabled.key());
static final ConfigKey<Long> KubernetesEtcdNodeSetupTimeout = new ConfigKey<Long>("Advanced", Long.class,
"cloud.kubernetes.etcd.node.setup.timeout",
"600",
"Timeout interval (in seconds) in which etcd should be running on each dedicated etcd node of a Kubernetes cluster that is starting. " +
"The cluster start fails when it is not, instead of waiting for the cluster start timeout",
true,
KubernetesServiceEnabled.key());

KubernetesCluster findById(final Long id);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,39 @@ protected void setupKubernetesEtcdNetworkRules(List<UserVm> etcdVms, Network net
}
}

/**
* The control nodes cannot bring up the API server without etcd, so a cluster whose etcd nodes failed to set up
* would only fail when the cluster start timeout is over. Check the etcd nodes first and fail with the reason.
*/
protected void verifyEtcdRunningOnEtcdNodes(final List<UserVm> etcdVms, final List<UserVm> clusterVMs, final Network network, final long startTimeoutTime) {
if (CollectionUtils.isEmpty(etcdVms)) {
return;
}
final long timeoutTime = Math.min(startTimeoutTime,
System.currentTimeMillis() + KubernetesClusterService.KubernetesEtcdNodeSetupTimeout.value() * 1000);
final boolean directAccess = manager.isDirectAccess(network);
final int etcdNodeStartPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value();
for (int i = 0; i < etcdVms.size(); i++) {
final UserVm etcdVm = etcdVms.get(i);
// Same route as the port forwarding rules of setupKubernetesEtcdNetworkRules
final String ipAddress = directAccess ? etcdVm.getPrivateIpAddress() : publicIpAddress;
final int port = directAccess ? DEFAULT_SSH_PORT : etcdNodeStartPort + i;
final String failure = waitForEtcdOnEtcdNode(etcdVm, ipAddress, port, timeoutTime);
if (failure != null) {
List<UserVm> vmsWithIso = new ArrayList<>(etcdVms);
vmsWithIso.addAll(clusterVMs);
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s as etcd is not running on etcd node %s: %s",
kubernetesCluster.getName(), etcdVm.getDisplayName(), failure), kubernetesCluster, vmsWithIso, KubernetesCluster.Event.CreateFailed, null);
}
}
logger.info("etcd is running on all {} etcd node(s) of Kubernetes cluster: {}", etcdVms.size(), kubernetesCluster);
}

protected String waitForEtcdOnEtcdNode(final UserVm etcdVm, final String ipAddress, final int port, final long timeoutTime) {
return KubernetesClusterUtil.waitForEtcdOnEtcdNode(kubernetesCluster, etcdVm.getDisplayName(), ipAddress, port,
CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime, 10000);
}

private void startKubernetesClusterVMs(Long domainId, Long accountId) {
List <UserVm> clusterVms = getKubernetesClusterVMs();
for (final UserVm vm : clusterVms) {
Expand Down Expand Up @@ -860,6 +893,7 @@ public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId, Lon
}
attachIsoKubernetesVMs(etcdVms);
attachIsoKubernetesVMs(clusterVMs);
verifyEtcdRunningOnEtcdNodes(etcdVms, clusterVMs, network, startTimeoutTime);
if (!KubernetesClusterUtil.isKubernetesClusterControlVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), startTimeoutTime)) {
String msg = String.format("Failed to setup Kubernetes cluster : %s is not in usable state as the system is unable to access control node VMs of the cluster", kubernetesCluster.getName());
if (kubernetesCluster.getControlNodeCount() > 1 && manager.isDirectAccess(network)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,59 @@ public static boolean isKubernetesClusterServerRunning(final KubernetesCluster k
return k8sApiServerSetup;
}

// Written by the setup script of an etcd node (conf/etcd-node.yml) with the reason when the setup fails
public static final String ETCD_NODE_SETUP_FAILED_FILE = "/home/cloud/etcd-setup-failed";
protected static final String ETCD_NODE_ACTIVE = "ACTIVE";
protected static final String ETCD_NODE_FAILED = "FAILED:";
protected static final String ETCD_NODE_STATUS_COMMAND = String.format(
"if [ -f %1$s ]; then echo \"%2$s $(cat %1$s)\"; elif sudo systemctl is-active --quiet etcd; then echo %3$s; else echo WAITING; fi",
ETCD_NODE_SETUP_FAILED_FILE, ETCD_NODE_FAILED, ETCD_NODE_ACTIVE);

protected static Pair<Boolean, String> getEtcdNodeStatus(final String ipAddress, final int port, final String user, final File sshKeyFile) throws Exception {
return SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, ETCD_NODE_STATUS_COMMAND, 10000, 10000, 20000);
}

/**
* Waits for etcd to run on a dedicated etcd node. etcd has to be seen running on two checks in a row, so that
* an etcd which keeps crashing and restarting is not taken as running.
*
* @return null once etcd is running, otherwise the reason why the node is not usable
*/
public static String waitForEtcdOnEtcdNode(final KubernetesCluster kubernetesCluster, final String nodeName, final String ipAddress,
final int port, final String user, final File sshKeyFile,
final long timeoutTime, final long waitDuration) {
String lastStatus = "the node was not reachable over SSH";
boolean activeOnLastCheck = false;
while (System.currentTimeMillis() < timeoutTime) {
boolean active = false;
try {
Pair<Boolean, String> result = getEtcdNodeStatus(ipAddress, port, user, sshKeyFile);
String output = result.second() == null ? "" : result.second().trim();
if (Boolean.TRUE.equals(result.first()) && output.startsWith(ETCD_NODE_FAILED)) {
return output.substring(ETCD_NODE_FAILED.length()).trim();
}
active = Boolean.TRUE.equals(result.first()) && output.startsWith(ETCD_NODE_ACTIVE);
lastStatus = active ? "etcd did not stay running" : "the etcd service was not running";
} catch (Exception e) {
lastStatus = "the node was not reachable over SSH";
LOGGER.debug("Unable to check etcd on node {} of Kubernetes cluster {}: {}", nodeName, kubernetesCluster, e.getMessage());
}
if (active && activeOnLastCheck) {
return null;
}
activeOnLastCheck = active;
LOGGER.info("Waiting for etcd to run on node {} of Kubernetes cluster {}", nodeName, kubernetesCluster);
try {
Thread.sleep(waitDuration);
} catch (InterruptedException ie) {
LOGGER.warn("Interrupted while waiting for etcd on node {} of Kubernetes cluster {}", nodeName, kubernetesCluster, ie);
Thread.currentThread().interrupt();
break;
}
}
return String.format("etcd did not start in time, on the last check %s", lastStatus);
}

public static boolean isKubernetesClusterControlVmRunning(final KubernetesCluster kubernetesCluster, final String ipAddress,
final int port, final long timeoutTime) {
boolean controlVmRunning = false;
Expand Down
Loading
Loading