Cli-testing migration : remove small dependencies

.github/scripts/end2end/setup-e2e-env.sh

@@ -157,7 +157,7 @@ else
     export STORAGE_ACCOUNT_OWNER_USER_NAME="storage_account_owner"
     export DATA_CONSUMER_USER_NAME="data_consumer"
     export DATA_ACCESSOR_USER_NAME="data_accessor"
-    # env vars used by cli-testing's Keycloak.ts seeder
+    # env vars read by tests/functional/ctst/steps/utils/keycloak.ts
     export ACCOUNT="${ZENKO_ACCOUNT_NAME}"
     export STORAGE_MANAGER="${STORAGE_MANAGER_USER_NAME}"
     export STORAGE_ACCOUNT_OWNER="${STORAGE_ACCOUNT_OWNER_USER_NAME}"

tests/functional/clients/azure.ts

@@ -9,20 +9,13 @@ import {
     QueueServiceClient,
     StorageSharedKeyCredential as StorageQueueSharedKeyCredential,
 } from '@azure/storage-queue';
+import { requireEnv } from './env';
 
 export type AzureCreds = {
     accountName: string;
     accountKey: string;
 };
 
-function requireEnv(name: string): string {
-    const value = process.env[name];
-    if (!value) {
-        throw new Error(`Missing required environment variable: ${name}`);
-    }
-    return value;
-}
-
 export default class AzureClient {
     private readonly blobClient: BlobServiceClient;
     private readonly queueClient: QueueServiceClient;

tests/functional/clients/env.ts

@@ -0,0 +1,7 @@
+export function requireEnv(name: string): string {
+    const value = process.env[name];
+    if (!value) {
+        throw new Error(`Missing required environment variable: ${name}`);
+    }
+    return value;
+}

tests/functional/ctst/common/common.ts

@@ -1,8 +1,8 @@
 import { ListObjectVersionsOutput } from '@aws-sdk/client-s3';
 import { Given, setDefaultTimeout, Then, When } from '@cucumber/cucumber';
-import { CacheHelper, Constants, Identity, IdentityEnum, S3, Utils } from 'cli-testing';
+import { CacheHelper, Identity, IdentityEnum, S3 } from 'cli-testing';
 import Zenko from 'world/Zenko';
-import { parseGoDuration, safeJsonParse } from './utils';
+import { parseGoDuration, safeJsonParse, sleep, randomString } from './utils';
 import assert from 'assert';
 import { Admin } from '@platformatic/kafka';
 import {
@@ -20,7 +20,7 @@ import {
 import { ActionPermissionsType } from 'steps/bucket-policies/utils';
 import constants from './constants';
 
-setDefaultTimeout(Constants.DEFAULT_TIMEOUT);
+setDefaultTimeout(constants.DEFAULT_TIMEOUT);
 
 /**
  * Cleans the created test bucket
@@ -71,7 +71,7 @@ async function addMultipleObjects(this: Zenko, numberObjects: number,
     for (let i = 1; i <= numberObjects; i++) {
         this.resetCommand();
         const objectNameFinal = getObjectNameWithBackendFlakiness.call(this, `${objectName}-${i}`) ||
-            Utils.randomString();
+            randomString();
         if (sizeBytes > 0) {
             this.addToSaved('objectSize', sizeBytes);
         }
@@ -144,16 +144,16 @@ async function createBucket(world: Zenko, versioning: string, bucketName: string
 
 Given('a {string} bucket with dot', async function (this: Zenko, versioning: string) {
     const preName = this.getSaved<string>('accountName') ||
-        this.parameters.AccountName || Constants.ACCOUNT_NAME;
+        this.parameters.AccountName || constants.ACCOUNT_NAME;
     await createBucket(this, versioning,
-        `${preName}.${Constants.BUCKET_NAME_TEST}${Utils.randomString()}`.toLocaleLowerCase());
+        `${preName}.${constants.BUCKET_NAME_TEST}${randomString()}`.toLocaleLowerCase());
 });
 
 Given('a {string} bucket', async function (this: Zenko, versioning: string) {
     const preName = this.getSaved<string>('accountName') ||
-        this.parameters.AccountName || Constants.ACCOUNT_NAME;
+        this.parameters.AccountName || constants.ACCOUNT_NAME;
     await createBucket(this, versioning,
-        `${preName}${Constants.BUCKET_NAME_TEST}${Utils.randomString()}`.toLocaleLowerCase());
+        `${preName}${constants.BUCKET_NAME_TEST}${randomString()}`.toLocaleLowerCase());
 });
 
 Given('an existing bucket {string} {string} versioning, {string} ObjectLock {string} retention mode', async function
@@ -381,7 +381,7 @@ Then('kafka consumed messages should not take too much place on disk', { timeout
                     break;
                 }
                 remainingTopics = nextTopics;
-                await Utils.sleep(5000);
+                await sleep(5000);
             }
         } finally {
             await kafkaAdmin.close();
@@ -431,16 +431,16 @@ When('the user tries to perform the current S3 action on the bucket {int} times
         for (let i = 0; i < numberOfRuns; i++) {
             // For repeated WRITE actions, we want to change the object name
             if (action.action === 'PutObject') {
-                this.addToSaved('objectName', `objectrepeat-${Utils.randomString()}`);
+                this.addToSaved('objectName', `objectrepeat-${randomString()}`);
             } else if (action.action === 'CopyObject') {
-                this.addToSaved('copyObject', `objectrepeatcopy-${Utils.randomString()}`);
+                this.addToSaved('copyObject', `objectrepeatcopy-${randomString()}`);
             }
             await runActionAgainstBucket(this, this.getSaved<ActionPermissionsType>('currentAction').action);
             if (this.getResult().err && this.getResult().retryable?.throttling !== true) {
                 this.logger.debug('Error during repeated action', { error: this.getResult().err });
                 break;
             }
-            await Utils.sleep(delay);
+            await sleep(delay);
         }
     });
 
@@ -492,6 +492,6 @@ When('I PUT an object with size {int}', async function (this: Zenko, size: numbe
         this.addToSaved('objectSize', size);
     }
     const result = await addMultipleObjects.call(
-        this, 1, `object-${Utils.randomString()}`, size);
+        this, 1, `object-${randomString()}`, size);
     this.setResult(result!);
 });

tests/functional/ctst/common/constants.ts

@@ -1,4 +1,34 @@
 export default {
     complianceRetention: 'COMPLIANCE',
     governanceRetention: 'GOVERNANCE',
+    DEFAULT_SUBDOMAIN: 'my-company.com',
+    DEFAULT_TIMEOUT: 100000,
+    ACCOUNT_NAME: 'AccountTest',
+    INTERNAL_SERVICES_ACCOUNT_ID: '000000000000',
+    USER_NAME_TEST: 'userNameForTest',
+    POLICY_NAME_TEST: 'policyNameForTest',
+    ROLE_NAME_TEST: 'roleNameForTest',
+    BUCKET_NAME_TEST: 'bucketForTest',
+    K_CLIENT: 'zenko-ui',
+    MAX_ACCOUNT_CHECK_RETRIES: 100,
+    // Must match the file name cli-testing's S3.getObject writes downloaded objects to
+    OUTFILE_NAME: 'out.loc',
+    assumeRolePolicy: JSON.stringify({
+        Version: '2012-10-17',
+        Statement: [
+            {
+                Effect: 'Allow',
+                Action: ['iam:ListRoles', 'sts:AssumeRole'],
+                Resource: '*',
+            },
+        ],
+    }),
+    assumeRoleTrustPolicy: JSON.stringify({
+        Version: '2012-10-17',
+        Statement: {
+            Effect: 'Allow',
+            Principal: '*',
+            Action: 'sts:AssumeRole',
+        },
+    }),
 };

tests/functional/ctst/common/utils.test.ts

@@ -1,5 +1,5 @@
 import assert from 'assert';
-import { paramToCli, parseGoDuration } from './utils';
+import { isAccessKeys, paramToCli, parseGoDuration } from './utils';
 
 const durations: [string, number][] = [
     ['1m', 60],
@@ -58,3 +58,20 @@ for (const [params, expected] of cliParams) {
         `paramToCli(${JSON.stringify(params)}) = "${result}", expected "${expected}"`,
     );
 }
+
+const accessKeysCases: [Record<string, string>, boolean][] = [
+    [{ accessKeyId: 'AK', secretAccessKey: 'SK' }, true],
+    [{ accessKeyId: 'AK', secretAccessKey: 'SK', sessionToken: 'ST' }, true],
+    [{ accessKeyId: 'AK', secretAccessKey: 'SK', sessionToken: '' }, false],
+    [{ accessKeyId: '', secretAccessKey: 'SK' }, false],
+    [{ accessKeyId: 'AK' }, false],
+    [{}, false],
+];
+
+for (const [creds, expected] of accessKeysCases) {
+    const result = isAccessKeys(creds);
+    assert.strictEqual(
+        result, expected,
+        `isAccessKeys(${JSON.stringify(creds)}) = ${result}, expected ${expected}`,
+    );
+}

tests/functional/ctst/common/utils.ts

@@ -1,7 +1,8 @@
 import assert from 'assert';
 import { exec } from 'child_process';
 import http from 'http';
-import { createHash } from 'crypto';
+import { createHash, randomBytes } from 'crypto';
+import { setTimeout as sleep } from 'timers/promises';
 import { Command, IAM, Identity, IdentityEnum } from 'cli-testing';
 import {
     AttachedPolicy,
@@ -15,10 +16,20 @@ import Zenko from 'world/Zenko';
 import fs from 'fs';
 import { runOnceAcrossWorkers } from 'common/WorkerCoordination';
 import { ITestCaseHookParameter } from '@cucumber/cucumber';
-import { AWSCredentials, Utils } from 'cli-testing';
+import { AWSCredentials } from 'cli-testing';
 import { createBucketWithConfiguration, putObject } from '../steps/utils/utils';
 import { createJobAndWaitForCompletion } from '../steps/utils/kubernetes';
 
+export { sleep };
+
+export function randomString(): string {
+    return randomBytes(5).toString('hex');
+}
+
+export function isAccessKeys(object: Partial<AWSCredentials>): object is AWSCredentials {
+    return Boolean(object.accessKeyId && object.secretAccessKey && object.sessionToken !== '');
+}
+
 export function assertBudgetFitsStepTimeout(budgetSeconds: number, stepTimeoutMs: number): void {
     assert.ok(
         budgetSeconds * 1000 < stepTimeoutMs,
@@ -417,7 +428,7 @@ export async function prepareMetricsScenarios(
             }
 
             await createJobAndWaitForCompletion(world, jobName, jobNamespace);
-            await Utils.sleep(2000);
+            await sleep(2000);
             fs.writeFileSync(dataFile, JSON.stringify(output));
         },
     );

tests/functional/ctst/steps/azureArchive.ts

@@ -1,9 +1,10 @@
 import fs from 'fs';
 import path from 'path';
 import assert from 'assert';
-import { safeJsonParse, request } from '../common/utils';
+import { safeJsonParse, request, sleep } from '../common/utils';
 import { Given, Then, When } from '@cucumber/cucumber';
-import { S3, Constants, Utils } from 'cli-testing';
+import { S3 } from 'cli-testing';
+import constants from 'common/constants';
 import util from 'util';
 import { exec } from 'child_process';
 import Zenko from 'world/Zenko';
@@ -56,7 +57,7 @@ async function isObjectRehydrated(zenko: Zenko, objectName: string) {
         if (found) {
             return tarName;
         }
-        await Utils.sleep(1000);
+        await sleep(1000);
     }
     return undefined;
 }
@@ -230,7 +231,7 @@ Then('object {string} should have the same data', async function (this: Zenko, o
     const res = await S3.getObject(this.getCommandParameters());
     assert.ifError(res.err);
     const cliTestingPath = path.dirname(require.resolve('cli-testing'));
-    const objectPath = path.join(cliTestingPath, 'utils', 'api', Constants.OUTFILE_NAME);
+    const objectPath = path.join(cliTestingPath, 'utils', 'api', constants.OUTFILE_NAME);
     const objectBuffer = fs.readFileSync(objectPath);
     fs.rmSync(objectPath);
     const expectedContent = Buffer.alloc(Buffer.byteLength(objectBuffer), 'a');
@@ -331,7 +332,7 @@ Then('the storage class of object {string} must stay {string} for {int} seconds'
             if (head?.StorageClass !== expectedClass) {
                 break;
             }
-            await Utils.sleep(1000);
+            await sleep(1000);
             secondsPassed++;
         }
         assert(secondsPassed === seconds);
@@ -360,7 +361,7 @@ When('i run sorbetctl to retry failed restore for {string} location',
 When('i wait for {int} days', { timeout: 10 * 60 * 1000 }, async function (this: Zenko, days: number) {
     const realTimeDay = days * 24 * 60 * 60 * 1000 /
         (this.parameters.TimeProgressionFactor > 1 ? this.parameters.TimeProgressionFactor : 1);
-    await Utils.sleep(realTimeDay);
+    await sleep(realTimeDay);
 });
 
 Then('object {string} should expire in {int} days', async function (this: Zenko, objectName: string, days: number) {

tests/functional/ctst/steps/bucket-policies/common.ts

@@ -19,8 +19,8 @@ import {
     runActionAgainstBucket,
 } from 'steps/utils/utils';
 import assert from 'assert';
-import { IAM, Identity, S3, Utils } from 'cli-testing';
-import { extractPropertyFromResults } from 'common/utils';
+import { IAM, Identity, S3 } from 'cli-testing';
+import { extractPropertyFromResults, randomString } from 'common/utils';
 
 Given('an action {string}', function (this: Zenko, apiName: string) {
     // dynamically know the config based on the action
@@ -52,7 +52,7 @@ Given('an existing bucket prepared for the action', async function (this: Zenko)
         this.getSaved<string>('withObjectLock'),
         this.getSaved<string>('retentionMode'));
     if (this.getSaved<boolean>('preExistingObject')) {
-        await putObject(this, `objectforbptests-${Utils.randomString()}`);
+        await putObject(this, `objectforbptests-${randomString()}`);
     }
 });
 
@@ -139,7 +139,7 @@ Given('an {string} IAM Policy that {string} with {string} effect for the current
     }
     const createdPolicy = await IAM.createPolicy({
         policyDocument: JSON.stringify(basePolicy),
-        policyName: `policyforauthz-${Utils.randomString()}`,
+        policyName: `policyforauthz-${randomString()}`,
     });
     const policyArn = extractPropertyFromResults<string>(createdPolicy, 'Policy', 'Arn');
 
@@ -347,7 +347,7 @@ Given('an environment setup for the API', async function (this: Zenko) {
     };
     const createdPolicy = await IAM.createPolicy({
         policyDocument: JSON.stringify(basePolicy),
-        policyName: `policyforauthz-${Utils.randomString()}`,
+        policyName: `policyforauthz-${randomString()}`,
     });
     const policyArn = extractPropertyFromResults<string>(createdPolicy, 'Policy', 'Arn');
 
@@ -378,7 +378,7 @@ Given('an environment setup for the API', async function (this: Zenko) {
     case 'CompleteMultipartUpload':
     case 'AbortMultipartUpload':
     case 'UploadPart':
-        const objectKey = `multipartUpload-${Utils.randomString()}`;
+        const objectKey = `multipartUpload-${randomString()}`;
         const initiateMPUResult = await S3.createMultipartUpload({
             bucket: this.getSaved<string>('bucketName'),
             key: objectKey,
@@ -389,11 +389,11 @@ Given('an environment setup for the API', async function (this: Zenko) {
         break;
     case 'UploadPartCopy':
         // create an object to copy from
-        const copyObjectKey = `objectforcopy-${Utils.randomString()}`;
+        const copyObjectKey = `objectforcopy-${randomString()}`;
         await putObject(this, copyObjectKey);
         this.addToSaved('objectName', copyObjectKey);
         // create an object for the MPU as copyObject
-        const objectKeyCopy = `multipartUpload-${Utils.randomString()}`;
+        const objectKeyCopy = `multipartUpload-${randomString()}`;
         const initiateMPUResultCopy = await S3.createMultipartUpload({
             bucket: this.getSaved<string>('bucketName'),
             key: objectKeyCopy,
@@ -423,7 +423,7 @@ Given('an environment setup for the API', async function (this: Zenko) {
         this.addCommandParameter({ bypassGovernanceRetention: 'true' });
         break;
     case 'CreateMultipartUpload':
-        this.addToSaved('objectName', `objectforbptests-${Utils.randomString()}`);
+        this.addToSaved('objectName', `objectforbptests-${randomString()}`);
         this.addCommandParameter({ key: this.getSaved<string>('objectName') });
         break;
     default:

tests/functional/ctst/steps/cloudserverAuth.ts

@@ -1,6 +1,8 @@
 import { When, Then } from '@cucumber/cucumber';
 import Zenko from '../world/Zenko';
-import { Utils, S3, Constants } from 'cli-testing';
+import { S3 } from 'cli-testing';
+import { randomString } from 'common/utils';
+import constants from 'common/constants';
 import { strict as assert } from 'assert';
 
 interface DeleteObjectsResult {
@@ -36,8 +38,8 @@ When('the user tries to perform CreateBucket', async function (this: Zenko) {
     this.resetCommand();
     this.useSavedIdentity();
     const preName = this.getSaved<string>('accountName') ||
-        this.parameters.AccountName || Constants.ACCOUNT_NAME;
-    const usedBucketName = `${preName}${Constants.BUCKET_NAME_TEST}${Utils.randomString()}`.toLocaleLowerCase();
+        this.parameters.AccountName || constants.ACCOUNT_NAME;
+    const usedBucketName = `${preName}${constants.BUCKET_NAME_TEST}${randomString()}`.toLocaleLowerCase();
     this.addToSaved('bucketName', usedBucketName);
     this.addCommandParameter({ bucket: usedBucketName });
     // Empty strings are used to pass parameters that are used as a flag and do not require a value