From 13040da8fc47356e1c6a1d5412b83a59830b3a5e Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Sat, 28 Mar 2026 06:37:16 +0000 Subject: [PATCH 1/6] feat: auto create credentials.yaml --- packages/cli/src/projects/checkout.ts | 4 + .../cli/src/projects/create-credentials.ts | 93 ++++++++++++++++ .../test/projects/create-credentials.test.ts | 104 ++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 packages/cli/src/projects/create-credentials.ts create mode 100644 packages/cli/test/projects/create-credentials.test.ts diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index 249aa52d9..5ae52a2f7 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -15,6 +15,7 @@ import { tidyWorkflowDir, updateForkedFrom, } from './util'; +import { createProjectCredentials } from './create-credentials'; export type CheckoutOptions = Pick< Opts, @@ -125,5 +126,8 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => { logger?.warn('WARNING! No content for file', f); } } + + createProjectCredentials(workspacePath, switchProject, logger); + logger?.success(`Expanded project to ${workspacePath}`); }; diff --git a/packages/cli/src/projects/create-credentials.ts b/packages/cli/src/projects/create-credentials.ts new file mode 100644 index 000000000..73a1fbef9 --- /dev/null +++ b/packages/cli/src/projects/create-credentials.ts @@ -0,0 +1,93 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import type Project from '@openfn/project'; +import { jsonToYaml, yamlToJson } from '@openfn/project'; + +import { CREDENTIALS_KEY } from '../execute/apply-credential-map'; +import type { Logger } from '../util/logger'; + +export function collectCredentialReferences(project: Project): string[] { + const ids = new Set(); + for (const wf of project.workflows) { + for (const step of wf.steps) { + const job = step as { + configuration?: string | Record | null; + }; + const { configuration } = job; + // picking credential + if (typeof configuration === 'string' && configuration.trim()) { + if (!configuration.endsWith('.json')) { + ids.add(configuration); + } + } else if ( + // picking from an obj + configuration && + typeof configuration === 'object' && + !Array.isArray(configuration) && + typeof configuration[CREDENTIALS_KEY] === 'string' && + (configuration[CREDENTIALS_KEY] as string).trim() + ) { + ids.add(configuration[CREDENTIALS_KEY] as string); + } + } + } + return Array.from(ids); +} + +export function createProjectCredentials( + workspacePath: string, + project: Project, + logger?: Logger +): void { + const credentialsPath = project.config.credentials; + if (typeof credentialsPath !== 'string' || !credentialsPath.trim()) return; + + const ids = collectCredentialReferences(project); + if (!ids.length) return; + + const absolutePath = path.resolve(workspacePath, credentialsPath); + let existing: Record = {}; + + if (fs.existsSync(absolutePath)) { + const raw = fs.readFileSync(absolutePath, 'utf8'); + if (raw.trim()) { + try { + if (credentialsPath.endsWith('.json')) { + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) + throw new Error('credential file contains invalid JSON'); + + existing = parsed as Record; + } else { + const parsed = yamlToJson(raw) as unknown; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + existing = parsed as Record; + } else if (parsed != null) { + throw new Error('credential file contains invalid YAML'); + } + } + } catch (e: any) { + throw new Error( + `Failed to parse credential at ${credentialsPath}: ${e?.message ?? e}` + ); + } + } + } + + const new_creds = ids.filter((id) => !(id in existing)).sort(); + if (!new_creds.length) return; + + const merged: Record = { ...existing }; + for (const id of new_creds) { + merged[id] = {}; + } + + const content = credentialsPath.endsWith('.json') + ? `${JSON.stringify(merged, null, 2)}` + : jsonToYaml(merged); + + fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); + fs.writeFileSync(absolutePath, content, 'utf8'); + logger?.debug(`Added ${new_creds.length} credentials to ${credentialsPath}`); +} diff --git a/packages/cli/test/projects/create-credentials.test.ts b/packages/cli/test/projects/create-credentials.test.ts new file mode 100644 index 000000000..f934f3e6d --- /dev/null +++ b/packages/cli/test/projects/create-credentials.test.ts @@ -0,0 +1,104 @@ +import test from 'ava'; +import fs from 'node:fs'; +import mock from 'mock-fs'; + +import Project from '@openfn/project'; +import { yamlToJson } from '@openfn/project'; + +import { CREDENTIALS_KEY } from '../../src/execute/apply-credential-map'; +import { + collectCredentialReferences, + createProjectCredentials, +} from '../../src/projects/create-credentials'; + +test.afterEach(() => { + try { + mock.restore(); + } catch { + } +}); + +const baseWorkflow = (steps: any[]) => ({ + id: 'wf', + name: 'wf', + history: [], + steps, +}); + +test('sync-credentials: inline string references', (t) => { + const project = new Project({ + id: 'p', + workflows: [ + baseWorkflow([ + { id: 'a', configuration: 'owner|cred' }, + { + id: 'b', + configuration: { [CREDENTIALS_KEY]: 'uuid-1', extra: true }, + }, + { id: 'c', configuration: 'ignored.json' }, + { id: 'd', configuration: '' }, + ]), + ], + } as any); + + const ids = collectCredentialReferences(project).sort(); + t.deepEqual(ids, ['owner|cred', 'uuid-1'].sort()); +}); + +test('sync-credentials: ignores duplicate references', (t) => { + const project = new Project({ + id: 'p', + workflows: [ + baseWorkflow([{ id: 'a', configuration: 'same' }]), + baseWorkflow([{ id: 'b', configuration: 'same' }]), + ], + } as any); + + t.deepEqual(collectCredentialReferences(project), ['same']); +}); + +test('sync-credentials: creates credential yaml file', (t) => { + mock({ '/ws': {} }); + + const project = new Project( + { + id: 'p', + workflows: [baseWorkflow([{ id: 'j', configuration: 'new-id' }])], + } as any, + { credentials: 'credentials.yaml' } + ); + + createProjectCredentials('/ws', project); + + t.true(fs.existsSync('/ws/credentials.yaml')); + const doc = yamlToJson(fs.readFileSync('/ws/credentials.yaml', 'utf8')) as any; + t.deepEqual(doc, { 'new-id': {} }); +}); + +test('sync-credentials: preserves existing credentials and adds missing ones', (t) => { + mock({ + '/ws': {}, + '/ws/credentials.yaml': `existing: + password: secret +`, + }); + + const project = new Project( + { + id: 'p', + workflows: [ + baseWorkflow([ + { id: 'j', configuration: 'existing' }, + { id: 'k', configuration: 'brand-new' }, + ]), + ], + } as any, + { credentials: 'credentials.yaml' } + ); + + createProjectCredentials('/ws', project); + + const doc = yamlToJson(fs.readFileSync('/ws/credentials.yaml', 'utf8')) as any; + t.is(doc.existing.password, 'secret'); + t.deepEqual(doc['brand-new'], {}); +}); From 17413fcb1fa2cc37c16b36a6d42a6a3259c6864f Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Mon, 30 Mar 2026 08:11:18 +0000 Subject: [PATCH 2/6] chore: update changelog --- .changeset/odd-fans-taste.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/odd-fans-taste.md diff --git a/.changeset/odd-fans-taste.md b/.changeset/odd-fans-taste.md new file mode 100644 index 000000000..f86399c23 --- /dev/null +++ b/.changeset/odd-fans-taste.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': minor +--- + +Auto generates credentials.yaml for pulled projects From b21752e4acecdfcb512eded0d02547661c551a52 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 1 Apr 2026 03:44:21 +0000 Subject: [PATCH 3/6] chore: resolve changes --- packages/cli/src/execute/handler.ts | 13 +---- .../cli/src/projects/create-credentials.ts | 56 +++++-------------- packages/cli/src/util/load-credential-map.ts | 23 ++++++++ .../test/projects/create-credentials.test.ts | 26 ++++----- 4 files changed, 51 insertions(+), 67 deletions(-) create mode 100644 packages/cli/src/util/load-credential-map.ts diff --git a/packages/cli/src/execute/handler.ts b/packages/cli/src/execute/handler.ts index 5df00afca..e005a5464 100644 --- a/packages/cli/src/execute/handler.ts +++ b/packages/cli/src/execute/handler.ts @@ -1,6 +1,4 @@ import type { ExecutionPlan } from '@openfn/lexicon'; -import { yamlToJson } from '@openfn/project'; -import { readFile } from 'node:fs/promises'; import path from 'node:path'; import type { ExecuteOptions } from './command'; @@ -22,6 +20,7 @@ import fuzzyMatchStep from '../util/fuzzy-match-step'; import abort from '../util/abort'; import validatePlan from '../util/validate-plan'; import overridePlanAdaptors from '../util/override-plan-adaptors'; +import { loadCredentialMap } from '../util/load-credential-map'; const matchStep = ( plan: ExecutionPlan, @@ -56,15 +55,9 @@ const loadAndApplyCredentialMap = async ( let creds = {}; if (options.credentials) { try { - const credsRaw = await readFile( - path.resolve(options.workspace!, options.credentials), - 'utf8' + creds = loadCredentialMap( + path.resolve(options.workspace!, options.credentials) ); - if (options.credentials.endsWith('.json')) { - creds = JSON.parse(credsRaw); - } else { - creds = yamlToJson(credsRaw); - } logger.info('Credential map loaded '); } catch (e: any) { // If we get here, the credential map failed to load diff --git a/packages/cli/src/projects/create-credentials.ts b/packages/cli/src/projects/create-credentials.ts index 73a1fbef9..edfeb5d0f 100644 --- a/packages/cli/src/projects/create-credentials.ts +++ b/packages/cli/src/projects/create-credentials.ts @@ -2,33 +2,23 @@ import fs from 'node:fs'; import path from 'node:path'; import type Project from '@openfn/project'; -import { jsonToYaml, yamlToJson } from '@openfn/project'; +import { jsonToYaml } from '@openfn/project'; -import { CREDENTIALS_KEY } from '../execute/apply-credential-map'; +import { loadCredentialMap } from '../util/load-credential-map'; import type { Logger } from '../util/logger'; -export function collectCredentialReferences(project: Project): string[] { +export function findCredentialIds(project: Project): string[] { const ids = new Set(); for (const wf of project.workflows) { for (const step of wf.steps) { - const job = step as { - configuration?: string | Record | null; - }; + const job = step as { configuration?: string | null }; const { configuration } = job; - // picking credential - if (typeof configuration === 'string' && configuration.trim()) { - if (!configuration.endsWith('.json')) { - ids.add(configuration); - } - } else if ( - // picking from an obj + if ( + typeof configuration === 'string' && configuration && - typeof configuration === 'object' && - !Array.isArray(configuration) && - typeof configuration[CREDENTIALS_KEY] === 'string' && - (configuration[CREDENTIALS_KEY] as string).trim() + !configuration.endsWith('.json') ) { - ids.add(configuration[CREDENTIALS_KEY] as string); + ids.add(configuration); } } } @@ -43,36 +33,16 @@ export function createProjectCredentials( const credentialsPath = project.config.credentials; if (typeof credentialsPath !== 'string' || !credentialsPath.trim()) return; - const ids = collectCredentialReferences(project); + const ids = findCredentialIds(project); if (!ids.length) return; const absolutePath = path.resolve(workspacePath, credentialsPath); let existing: Record = {}; - if (fs.existsSync(absolutePath)) { - const raw = fs.readFileSync(absolutePath, 'utf8'); - if (raw.trim()) { - try { - if (credentialsPath.endsWith('.json')) { - const parsed = JSON.parse(raw) as unknown; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) - throw new Error('credential file contains invalid JSON'); - - existing = parsed as Record; - } else { - const parsed = yamlToJson(raw) as unknown; - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { - existing = parsed as Record; - } else if (parsed != null) { - throw new Error('credential file contains invalid YAML'); - } - } - } catch (e: any) { - throw new Error( - `Failed to parse credential at ${credentialsPath}: ${e?.message ?? e}` - ); - } - } + try { + existing = loadCredentialMap(absolutePath); + } catch (e: any) { + // project doesn't have credential } const new_creds = ids.filter((id) => !(id in existing)).sort(); diff --git a/packages/cli/src/util/load-credential-map.ts b/packages/cli/src/util/load-credential-map.ts new file mode 100644 index 000000000..f38e5b2a7 --- /dev/null +++ b/packages/cli/src/util/load-credential-map.ts @@ -0,0 +1,23 @@ +import fs from 'node:fs'; +import { yamlToJson } from '@openfn/project'; + +export function loadCredentialMap(filePath: string): Record { + const raw = fs.readFileSync(filePath, 'utf8'); + if (!raw.trim()) return {}; + + if (filePath.endsWith('.json')) { + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('credential file contains invalid JSON'); + } + return parsed as Record; + } else { + const parsed = yamlToJson(raw) as unknown; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record; + } else if (parsed != null) { + throw new Error('credential file contains invalid YAML'); + } + return {}; + } +} diff --git a/packages/cli/test/projects/create-credentials.test.ts b/packages/cli/test/projects/create-credentials.test.ts index f934f3e6d..f1d1c6277 100644 --- a/packages/cli/test/projects/create-credentials.test.ts +++ b/packages/cli/test/projects/create-credentials.test.ts @@ -5,17 +5,15 @@ import mock from 'mock-fs'; import Project from '@openfn/project'; import { yamlToJson } from '@openfn/project'; -import { CREDENTIALS_KEY } from '../../src/execute/apply-credential-map'; import { - collectCredentialReferences, + findCredentialIds, createProjectCredentials, } from '../../src/projects/create-credentials'; test.afterEach(() => { try { mock.restore(); - } catch { - } + } catch {} }); const baseWorkflow = (steps: any[]) => ({ @@ -31,18 +29,14 @@ test('sync-credentials: inline string references', (t) => { workflows: [ baseWorkflow([ { id: 'a', configuration: 'owner|cred' }, - { - id: 'b', - configuration: { [CREDENTIALS_KEY]: 'uuid-1', extra: true }, - }, { id: 'c', configuration: 'ignored.json' }, { id: 'd', configuration: '' }, ]), ], } as any); - const ids = collectCredentialReferences(project).sort(); - t.deepEqual(ids, ['owner|cred', 'uuid-1'].sort()); + const ids = findCredentialIds(project); + t.deepEqual(ids, ['owner|cred']); }); test('sync-credentials: ignores duplicate references', (t) => { @@ -54,10 +48,10 @@ test('sync-credentials: ignores duplicate references', (t) => { ], } as any); - t.deepEqual(collectCredentialReferences(project), ['same']); + t.deepEqual(findCredentialIds(project), ['same']); }); -test('sync-credentials: creates credential yaml file', (t) => { +test.only('sync-credentials: creates credential yaml file', (t) => { mock({ '/ws': {} }); const project = new Project( @@ -71,7 +65,9 @@ test('sync-credentials: creates credential yaml file', (t) => { createProjectCredentials('/ws', project); t.true(fs.existsSync('/ws/credentials.yaml')); - const doc = yamlToJson(fs.readFileSync('/ws/credentials.yaml', 'utf8')) as any; + const doc = yamlToJson( + fs.readFileSync('/ws/credentials.yaml', 'utf8') + ) as any; t.deepEqual(doc, { 'new-id': {} }); }); @@ -98,7 +94,9 @@ test('sync-credentials: preserves existing credentials and adds missing ones', ( createProjectCredentials('/ws', project); - const doc = yamlToJson(fs.readFileSync('/ws/credentials.yaml', 'utf8')) as any; + const doc = yamlToJson( + fs.readFileSync('/ws/credentials.yaml', 'utf8') + ) as any; t.is(doc.existing.password, 'secret'); t.deepEqual(doc['brand-new'], {}); }); From 4f0cb595f733e5e8875da77b25906e9b21fadaa8 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 1 Apr 2026 03:46:55 +0000 Subject: [PATCH 4/6] chore: remove .trim() --- packages/cli/src/projects/create-credentials.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/projects/create-credentials.ts b/packages/cli/src/projects/create-credentials.ts index edfeb5d0f..0deed73cc 100644 --- a/packages/cli/src/projects/create-credentials.ts +++ b/packages/cli/src/projects/create-credentials.ts @@ -31,7 +31,7 @@ export function createProjectCredentials( logger?: Logger ): void { const credentialsPath = project.config.credentials; - if (typeof credentialsPath !== 'string' || !credentialsPath.trim()) return; + if (typeof credentialsPath !== 'string') return; const ids = findCredentialIds(project); if (!ids.length) return; From 4ef79ee7940827b6f5bff9d9905bfbec7ae81437 Mon Sep 17 00:00:00 2001 From: Farhan Yahaya Date: Wed, 1 Apr 2026 17:15:10 +0000 Subject: [PATCH 5/6] chore: update changelog --- .changeset/fine-seas-divide.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/fine-seas-divide.md diff --git a/.changeset/fine-seas-divide.md b/.changeset/fine-seas-divide.md new file mode 100644 index 000000000..e9edba887 --- /dev/null +++ b/.changeset/fine-seas-divide.md @@ -0,0 +1,7 @@ +--- +'@openfn/engine-multi': minor +'@openfn/runtime': minor +'@openfn/cli': minor +--- + +Auto create credentials.yaml for projects From 3553a9ed17a3442a4d056fa89cfc3ec08ecbc1dc Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 2 Apr 2026 12:14:42 +0100 Subject: [PATCH 6/6] remove invalid changeset --- .changeset/fine-seas-divide.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .changeset/fine-seas-divide.md diff --git a/.changeset/fine-seas-divide.md b/.changeset/fine-seas-divide.md deleted file mode 100644 index e9edba887..000000000 --- a/.changeset/fine-seas-divide.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@openfn/engine-multi': minor -'@openfn/runtime': minor -'@openfn/cli': minor ---- - -Auto create credentials.yaml for projects