Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 0 additions & 26 deletions .github/scripts/end2end/setup-e2e-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -146,26 +146,6 @@ else
export SEED_KEYCLOAK_DEFAULT_ROLES=true
export ZENKO_PORT="80"

# PRA admin credentials (may not exist for non-PRA runs; ignore errors)
export ADMIN_PRA_ACCESS_KEY_ID=$(kubectl get secret ${ZENKO_NAME}-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' 2>/dev/null | base64 -d 2>/dev/null || echo "")
export ADMIN_PRA_SECRET_ACCESS_KEY=$(kubectl get secret ${ZENKO_NAME}-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.secretKey}' 2>/dev/null | base64 -d 2>/dev/null || echo "")

# --- 11. Service user credentials ---
BACKBEAT_LCBP_1_CREDS=$(kubectl get secret -l app.kubernetes.io/name=backbeat-lcbp-user-creds,app.kubernetes.io/instance=${ZENKO_NAME} -o jsonpath='{.items[0].data.backbeat-lifecycle-bp-1\.json}' | base64 -d)
BACKBEAT_LCC_1_CREDS=$(kubectl get secret -l app.kubernetes.io/name=backbeat-lcc-user-creds,app.kubernetes.io/instance=${ZENKO_NAME} -o jsonpath='{.items[0].data.backbeat-lifecycle-conductor-1\.json}' | base64 -d)
BACKBEAT_LCOP_1_CREDS=$(kubectl get secret -l app.kubernetes.io/name=backbeat-lcop-user-creds,app.kubernetes.io/instance=${ZENKO_NAME} -o jsonpath='{.items[0].data.backbeat-lifecycle-op-1\.json}' | base64 -d)
BACKBEAT_QP_1_CREDS=$(kubectl get secret -l app.kubernetes.io/name=backbeat-qp-user-creds,app.kubernetes.io/instance=${ZENKO_NAME} -o jsonpath='{.items[0].data.backbeat-qp-1\.json}' | base64 -d)
SORBET_FWD_2_ACCESSKEY=$(kubectl get secret -l app.kubernetes.io/name=sorbet-fwd-creds,app.kubernetes.io/instance=${ZENKO_NAME} -o jsonpath='{.items[0].data.accessKey}' | base64 -d)
SORBET_FWD_2_SECRETKEY=$(kubectl get secret -l app.kubernetes.io/name=sorbet-fwd-creds,app.kubernetes.io/instance=${ZENKO_NAME} -o jsonpath='{.items[0].data.secretKey}' | base64 -d)
export SERVICE_USERS_CREDENTIALS=$(echo '{"backbeat-lifecycle-bp-1":'"${BACKBEAT_LCBP_1_CREDS}"',"backbeat-lifecycle-conductor-1":'"${BACKBEAT_LCC_1_CREDS}"',"backbeat-lifecycle-op-1":'"${BACKBEAT_LCOP_1_CREDS}"',"backbeat-qp-1":'"${BACKBEAT_QP_1_CREDS}"',"sorbet-fwd-2":{"accessKey":"'"${SORBET_FWD_2_ACCESSKEY}"'","secretKey":"'"${SORBET_FWD_2_SECRETKEY}"'"}}' | jq -R)

# --- 12. Kafka topics for sorbet ---
SORBET_CONFIG=$(kubectl get secret -l app.kubernetes.io/name=cold-sorbet-config-e2e-azure-archive,app.kubernetes.io/instance=${ZENKO_NAME} \
-o jsonpath='{.items[0].data.config\.json}' | base64 -di)
export KAFKA_DEAD_LETTER_TOPIC=$(echo "${SORBET_CONFIG}" | jq -r '."kafka-dead-letter-topic"')
export KAFKA_OBJECT_TASK_TOPIC=$(echo "${SORBET_CONFIG}" | jq -r '."kafka-object-task-topic"')
export KAFKA_GC_REQUEST_TOPIC=$(echo "${SORBET_CONFIG}" | jq -r '."kafka-gc-request-topic"')

# --- 13. Kafka host from backbeat config + port-forward ---
KAFKA_HOST_PORT_ORIG=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=${ZENKO_NAME} \
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq -r .kafka.hosts)
Expand Down Expand Up @@ -317,7 +297,6 @@ else
"StorageAccountOwnerUsername":"${STORAGE_ACCOUNT_OWNER_USER_NAME}",
"DataConsumerUsername":"${DATA_CONSUMER_USER_NAME}",
"DataAccessorUsername":"${DATA_ACCESSOR_USER_NAME}",
"ServiceUsersCredentials":${SERVICE_USERS_CREDENTIALS},
"AzureAccountName":"${AZURE_ACCOUNT_NAME}",
"AzureAccountKey":"${AZURE_SECRET_KEY}",
"AzureArchiveContainer":"${AZURE_ARCHIVE_BUCKET_NAME}",
Expand All @@ -326,16 +305,11 @@ else
"AzureArchiveManifestTier":"${AZURE_ARCHIVE_MANIFEST_ACCESS_TIER}",
"AzureArchiveQueue":"${AZURE_ARCHIVE_QUEUE_NAME:-}",
"TimeProgressionFactor":"${TIME_PROGRESSION_FACTOR}",
"KafkaObjectTaskTopic":"${KAFKA_OBJECT_TASK_TOPIC}",
"KafkaGCRequestTopic":"${KAFKA_GC_REQUEST_TOPIC}",
"KafkaDeadLetterQueueTopic":"${KAFKA_DEAD_LETTER_TOPIC}",
"InstanceID":"${INSTANCE_ID}",
"BackbeatApiHost":"${BACKBEAT_API_HOST}",
"BackbeatApiPort":"${BACKBEAT_API_PORT}",
"KafkaCleanerInterval":"${KAFKA_CLEANER_INTERVAL}",
"SorbetdRestoreTimeout":"${SORBETD_RESTORE_TIMEOUT}",
"DRAdminAccessKey":"${ADMIN_PRA_ACCESS_KEY_ID}",
"DRAdminSecretKey":"${ADMIN_PRA_SECRET_ACCESS_KEY}",
"UtilizationServiceHost":"${UTILIZATION_SERVICE_HOST}",
"UtilizationServicePort":"${UTILIZATION_SERVICE_PORT}",
"KubeconfigPath":"${KUBECONFIG:-${HOME}/.kube/config}"
Expand Down
6 changes: 3 additions & 3 deletions tests/ctst/steps/azureArchive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -360,9 +360,9 @@ When('i run sorbetctl to retry failed restore for {string} location',
{ timeout: 10 * 60 * 1000 }, async function (this: Zenko, location: string) {
const command = `./sorbetctl forward list failed --trigger-retry --skip-invalid \
--limit 300 \
--kafka-dead-letter-topic=${this.parameters.KafkaDeadLetterQueueTopic} \
--kafka-object-task-topic=${this.parameters.KafkaObjectTaskTopic} \
--kafka-gc-request-topic=${this.parameters.KafkaGCRequestTopic} \
--kafka-dead-letter-topic=${Zenko.kafkaTopics.deadLetterQueue} \
--kafka-object-task-topic=${Zenko.kafkaTopics.objectTask} \
--kafka-gc-request-topic=${Zenko.kafkaTopics.gcRequest} \
--kafka-brokers ${this.parameters.KafkaHosts}`;
try {
this.logger.debug('Running command', { command, location });
Expand Down
2 changes: 1 addition & 1 deletion tests/ctst/steps/dr/drctl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ export default class ZenkoDrctl {
}

private getKubeconfigPath(): string | undefined {
const kp = this.world.parameters.KubeconfigPath as string | undefined;
const kp = this.world.parameters.KubeconfigPath;
return kp || process.env.KUBECONFIG;
}

Expand Down
3 changes: 2 additions & 1 deletion tests/ctst/steps/reporting/storageUsageReporting.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ interface ReportingUsageResponse {
}

Given('an identity with the {string} keycloak persona', function (this: Zenko, persona: string) {
const username = (this.parameters as Record<string, string>)[persona] || persona;
const params = this.parameters as unknown as Record<string, string>;
const username = params[persona] || persona;
this.addToSaved('keycloakPersona', username);
});

Expand Down
45 changes: 44 additions & 1 deletion tests/ctst/steps/utils/kubernetes.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import fs from 'fs';
import * as path from 'path';
import { KubernetesHelper, Utils } from 'cli-testing';
import { CacheHelper, KubernetesHelper, Utils } from 'cli-testing';
import Zenko from 'world/Zenko';
import {
V1Job,
Expand Down Expand Up @@ -534,6 +534,49 @@ export async function createSecret(
}
}

export async function getSecretByLabel(
labelSelector: string,
dataField: string,
namespace = 'default',
): Promise<string> {
if (!KubernetesHelper.clientCore) {
KubernetesHelper.init(CacheHelper.parameters);
}
const coreClient = KubernetesHelper.clientCore as CoreV1Api;
const secretList = await coreClient.listNamespacedSecret({
namespace,
labelSelector,
});
const secret = secretList.items[0];
if (!secret?.data?.[dataField]) {
throw new Error(
`Secret field "${dataField}" not found for label "${labelSelector}"`,
);
}
return Buffer.from(secret.data[dataField], 'base64').toString('utf-8');
}

export async function getSecretField(
secretName: string,
dataField: string,
namespace = 'default',
): Promise<string> {
if (!KubernetesHelper.clientCore) {
KubernetesHelper.init(CacheHelper.parameters);
}
const coreClient = KubernetesHelper.clientCore as CoreV1Api;
const secret = await coreClient.readNamespacedSecret({
name: secretName, namespace,
});
const value = secret.data?.[dataField];
if (!value) {
throw new Error(
`Secret field "${dataField}" not found in secret "${secretName}"`,
);
}
return Buffer.from(value, 'base64').toString('utf-8');
}

export async function getMongoDBConfig(
world: Zenko,
namespace = 'default',
Expand Down
126 changes: 83 additions & 43 deletions tests/ctst/world/Zenko.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,10 @@ import {
} from 'cli-testing';

import { extractPropertyFromResults } from '../common/utils';
import { getSecretByLabel, getSecretField } from 'steps/utils/kubernetes';
import ZenkoDrctl from 'steps/dr/drctl';
import assert from 'assert';

interface ServiceUsersCredentials {
accessKey: string;
secretKey: string;
}

// Zenko entities
export interface SavedIdentity {
identityName: string;
Expand All @@ -54,8 +50,8 @@ export interface ZenkoWorldParameters extends ClientOptions {
AccountName: string;
AccountAccessKey: string;
AccountSecretKey: string;
DRAdminAccessKey?: string;
DRAdminSecretKey?: string;
AdminAccessKey: string;
AdminSecretKey: string;
DRSubdomain?: string;
VaultAuthHost: string;
NotificationDestination: string;
Expand Down Expand Up @@ -83,7 +79,6 @@ export interface ZenkoWorldParameters extends ClientOptions {
StorageAccountOwnerUsername: string;
DataConsumerUsername: string;
DataAccessorUsername: string;
ServiceUsersCredentials: string;
KeycloakTestPassword: string;
AzureAccountName: string;
AzureAccountKey: string;
Expand All @@ -93,17 +88,14 @@ export interface ZenkoWorldParameters extends ClientOptions {
AzureArchiveManifestTier: string;
AzureArchiveQueue: string;
TimeProgressionFactor: number;
KafkaDeadLetterQueueTopic: string;
KafkaObjectTaskTopic: string;
KafkaGCRequestTopic: string;
InstanceID: string;
BackbeatApiHost: string;
BackbeatApiPort: string;
KafkaCleanerInterval: string;
SorbetdRestoreTimeout: string;
UtilizationServiceHost: string;
UtilizationServicePort: string;
[key: string]: unknown;
KubeconfigPath?: string;
}

/**
Expand Down Expand Up @@ -137,26 +129,19 @@ export default class Zenko extends World<ZenkoWorldParameters> {
static readonly SECONDARY_SITE_NAME = 'dradmin';
static readonly PRA_INSTALL_COUNT_KEY = 'praInstallCount';

static kafkaTopics: {
deadLetterQueue: string;
objectTask: string;
gcRequest: string;
};

/**
* @constructor
* @param {Object} options - parameters provided as a CLI parameter when running the tests
*/
constructor(options: IWorldOptions<ZenkoWorldParameters>) {
super(options);
Logger.createLogger(this);
// store service users credentials from world parameters
if (this.parameters.ServiceUsersCredentials) {
const serviceUserCredentials =
JSON.parse(this.parameters.ServiceUsersCredentials) as Record<string, ServiceUsersCredentials>;
for (const serviceUserName in serviceUserCredentials) {
if (!Identity.hasIdentity(IdentityEnum.SERVICE_USER, serviceUserName, this.parameters.AccountName)) {
Identity.addIdentity(IdentityEnum.SERVICE_USER, serviceUserName, {
accessKeyId: serviceUserCredentials[serviceUserName].accessKey,
secretAccessKey: serviceUserCredentials[serviceUserName].secretKey,
}, this.parameters.AccountName);
}
}
}

// Workaround to be able to access global parameters in BeforeAll/AfterAll hooks
CacheHelper.cacheParameters({
Expand Down Expand Up @@ -191,29 +176,11 @@ export default class Zenko extends World<ZenkoWorldParameters> {
};
}

if (this.needsSecondarySite()) {
if (!Identity.hasIdentity(IdentityEnum.ADMIN, Zenko.SECONDARY_SITE_NAME)) {
Identity.addIdentity(IdentityEnum.ADMIN, Zenko.SECONDARY_SITE_NAME, {
accessKeyId: this.parameters.DRAdminAccessKey!,
secretAccessKey: this.parameters.DRAdminSecretKey!,
}, undefined, undefined, undefined, this.parameters.DRSubdomain);
}

Zenko.sites['sink'] = {
accountName: `dr${this.parameters.AccountName}`,
adminIdentityName: Zenko.SECONDARY_SITE_NAME,
};
}

this.logger.debug('Zenko sites', {
sites: Zenko.sites,
});
}

private needsSecondarySite() {
return this.parameters.DRAdminAccessKey && this.parameters.DRAdminSecretKey && this.parameters.DRSubdomain;
}

/**
* This function will dynamically determine if the result from the AWS command
* is a success or a failure. Based on the fact that AWS either return an empty string
Expand Down Expand Up @@ -630,6 +597,74 @@ export default class Zenko extends World<ZenkoWorldParameters> {
this.saveIdentityInformation(roleName, IdentityEnum.ASSUMED_ROLE, Identity.getCurrentAccountName());
}

private static async loadServiceUsers(accountName: string) {
const instanceSelector = 'app.kubernetes.io/instance=end2end';
const serviceUsers: Record<string, string> = {
'backbeat-lcbp-user-creds': 'backbeat-lifecycle-bp-1',
'backbeat-lcc-user-creds': 'backbeat-lifecycle-conductor-1',
'backbeat-lcop-user-creds': 'backbeat-lifecycle-op-1',
'backbeat-qp-user-creds': 'backbeat-qp-1',
};

for (const [secretName, userName] of Object.entries(serviceUsers)) {
const labelSelector =`app.kubernetes.io/name=${secretName},${instanceSelector}`;
const raw = await getSecretByLabel(labelSelector, `${userName}.json`);
const parsed = JSON.parse(raw) as { accessKey: string; secretKey: string };
Identity.addIdentity(IdentityEnum.SERVICE_USER, userName, {
accessKeyId: parsed.accessKey,
secretAccessKey: parsed.secretKey,
}, accountName);
}

const sorbetSelector =
`app.kubernetes.io/name=sorbet-fwd-creds,${instanceSelector}`;
const accessKey = await getSecretByLabel(sorbetSelector, 'accessKey');
const secretKey = await getSecretByLabel(sorbetSelector, 'secretKey');
Identity.addIdentity(IdentityEnum.SERVICE_USER, 'sorbet-fwd-2', {
accessKeyId: accessKey,
secretAccessKey: secretKey,
}, accountName);
}

private static async loadSorbetConfig() {
const labelSelector =
'app.kubernetes.io/name=cold-sorbet-config-e2e-azure-archive' +
',app.kubernetes.io/instance=end2end';
const raw = await getSecretByLabel(labelSelector, 'config.json');
const config = JSON.parse(raw) as Record<string, string>;
Zenko.kafkaTopics = {
deadLetterQueue: config['kafka-dead-letter-topic'],
objectTask: config['kafka-object-task-topic'],
gcRequest: config['kafka-gc-request-topic'],
};
}

private static async loadPRACredentials(parameters: ZenkoWorldParameters) {
if (!parameters.DRSubdomain) {
return;
}
const secretName = 'end2end-pra-management-vault-admin-creds.v1';
let accessKey: string;
let secretKey: string;
try {
accessKey = await getSecretField(secretName, 'accessKey');
secretKey = await getSecretField(secretName, 'secretKey');
} catch {
// PRA secret does not exist — fine for non-PRA tests
return;
}
if (!Identity.hasIdentity(IdentityEnum.ADMIN, Zenko.SECONDARY_SITE_NAME)) {
Identity.addIdentity(IdentityEnum.ADMIN, Zenko.SECONDARY_SITE_NAME, {
accessKeyId: accessKey,
secretAccessKey: secretKey,
}, undefined, undefined, undefined, parameters.DRSubdomain);
}
Zenko.sites['sink'] = {
accountName: `dr${parameters.AccountName}`,
adminIdentityName: Zenko.SECONDARY_SITE_NAME,
};
}

/**
* Hook Zenko is a utility function to prepare a Zenko
* @param {Object.<string,*>} parameters - the client-provided parameters
Expand All @@ -639,6 +674,11 @@ export default class Zenko extends World<ZenkoWorldParameters> {
CacheHelper.logger.debug('Initializing Zenko', {
parameters,
});

await Zenko.loadServiceUsers(parameters.AccountName);
await Zenko.loadSorbetConfig();
await Zenko.loadPRACredentials(parameters);

// Create the default account for each site configured
// and generate access keys for it
for (const siteKey in Zenko.sites) {
Expand Down
Loading