diff --git a/.changeset/fine-seas-divide.md b/.changeset/fine-seas-divide.md new file mode 100644 index 000000000..e9edba887 --- /dev/null +++ b/.changeset/fine-seas-divide.md @@ -0,0 +1,7 @@ +--- +'@openfn/engine-multi': minor +'@openfn/runtime': minor +'@openfn/cli': minor +--- + +Auto create credentials.yaml for projects diff --git a/.changeset/odd-fans-taste.md b/.changeset/odd-fans-taste.md new file mode 100644 index 000000000..f86399c23 --- /dev/null +++ b/.changeset/odd-fans-taste.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': minor +--- + +Auto generates credentials.yaml for pulled projects diff --git a/packages/cli/src/execute/handler.ts b/packages/cli/src/execute/handler.ts index 5df00afca..e005a5464 100644 --- a/packages/cli/src/execute/handler.ts +++ b/packages/cli/src/execute/handler.ts @@ -1,6 +1,4 @@ import type { ExecutionPlan } from '@openfn/lexicon'; -import { yamlToJson } from '@openfn/project'; -import { readFile } from 'node:fs/promises'; import path from 'node:path'; import type { ExecuteOptions } from './command'; @@ -22,6 +20,7 @@ import fuzzyMatchStep from '../util/fuzzy-match-step'; import abort from '../util/abort'; import validatePlan from '../util/validate-plan'; import overridePlanAdaptors from '../util/override-plan-adaptors'; +import { loadCredentialMap } from '../util/load-credential-map'; const matchStep = ( plan: ExecutionPlan, @@ -56,15 +55,9 @@ const loadAndApplyCredentialMap = async ( let creds = {}; if (options.credentials) { try { - const credsRaw = await readFile( - path.resolve(options.workspace!, options.credentials), - 'utf8' + creds = loadCredentialMap( + path.resolve(options.workspace!, options.credentials) ); - if (options.credentials.endsWith('.json')) { - creds = JSON.parse(credsRaw); - } else { - creds = yamlToJson(credsRaw); - } logger.info('Credential map loaded '); } catch (e: any) { // If we get here, the credential map failed to load diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index 249aa52d9..5ae52a2f7 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -15,6 +15,7 @@ import { tidyWorkflowDir, updateForkedFrom, } from './util'; +import { createProjectCredentials } from './create-credentials'; export type CheckoutOptions = Pick< Opts, @@ -125,5 +126,8 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => { logger?.warn('WARNING! No content for file', f); } } + + createProjectCredentials(workspacePath, switchProject, logger); + logger?.success(`Expanded project to ${workspacePath}`); }; diff --git a/packages/cli/src/projects/create-credentials.ts b/packages/cli/src/projects/create-credentials.ts new file mode 100644 index 000000000..0deed73cc --- /dev/null +++ b/packages/cli/src/projects/create-credentials.ts @@ -0,0 +1,63 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import type Project from '@openfn/project'; +import { jsonToYaml } from '@openfn/project'; + +import { loadCredentialMap } from '../util/load-credential-map'; +import type { Logger } from '../util/logger'; + +export function findCredentialIds(project: Project): string[] { + const ids = new Set(); + for (const wf of project.workflows) { + for (const step of wf.steps) { + const job = step as { configuration?: string | null }; + const { configuration } = job; + if ( + typeof configuration === 'string' && + configuration && + !configuration.endsWith('.json') + ) { + ids.add(configuration); + } + } + } + return Array.from(ids); +} + +export function createProjectCredentials( + workspacePath: string, + project: Project, + logger?: Logger +): void { + const credentialsPath = project.config.credentials; + if (typeof credentialsPath !== 'string') return; + + const ids = findCredentialIds(project); + if (!ids.length) return; + + const absolutePath = path.resolve(workspacePath, credentialsPath); + let existing: Record = {}; + + try { + existing = loadCredentialMap(absolutePath); + } catch (e: any) { + // project doesn't have credential + } + + const new_creds = ids.filter((id) => !(id in existing)).sort(); + if (!new_creds.length) return; + + const merged: Record = { ...existing }; + for (const id of new_creds) { + merged[id] = {}; + } + + const content = credentialsPath.endsWith('.json') + ? `${JSON.stringify(merged, null, 2)}` + : jsonToYaml(merged); + + fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); + fs.writeFileSync(absolutePath, content, 'utf8'); + logger?.debug(`Added ${new_creds.length} credentials to ${credentialsPath}`); +} diff --git a/packages/cli/src/util/load-credential-map.ts b/packages/cli/src/util/load-credential-map.ts new file mode 100644 index 000000000..f38e5b2a7 --- /dev/null +++ b/packages/cli/src/util/load-credential-map.ts @@ -0,0 +1,23 @@ +import fs from 'node:fs'; +import { yamlToJson } from '@openfn/project'; + +export function loadCredentialMap(filePath: string): Record { + const raw = fs.readFileSync(filePath, 'utf8'); + if (!raw.trim()) return {}; + + if (filePath.endsWith('.json')) { + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('credential file contains invalid JSON'); + } + return parsed as Record; + } else { + const parsed = yamlToJson(raw) as unknown; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record; + } else if (parsed != null) { + throw new Error('credential file contains invalid YAML'); + } + return {}; + } +} diff --git a/packages/cli/test/projects/create-credentials.test.ts b/packages/cli/test/projects/create-credentials.test.ts new file mode 100644 index 000000000..f1d1c6277 --- /dev/null +++ b/packages/cli/test/projects/create-credentials.test.ts @@ -0,0 +1,102 @@ +import test from 'ava'; +import fs from 'node:fs'; +import mock from 'mock-fs'; + +import Project from '@openfn/project'; +import { yamlToJson } from '@openfn/project'; + +import { + findCredentialIds, + createProjectCredentials, +} from '../../src/projects/create-credentials'; + +test.afterEach(() => { + try { + mock.restore(); + } catch {} +}); + +const baseWorkflow = (steps: any[]) => ({ + id: 'wf', + name: 'wf', + history: [], + steps, +}); + +test('sync-credentials: inline string references', (t) => { + const project = new Project({ + id: 'p', + workflows: [ + baseWorkflow([ + { id: 'a', configuration: 'owner|cred' }, + { id: 'c', configuration: 'ignored.json' }, + { id: 'd', configuration: '' }, + ]), + ], + } as any); + + const ids = findCredentialIds(project); + t.deepEqual(ids, ['owner|cred']); +}); + +test('sync-credentials: ignores duplicate references', (t) => { + const project = new Project({ + id: 'p', + workflows: [ + baseWorkflow([{ id: 'a', configuration: 'same' }]), + baseWorkflow([{ id: 'b', configuration: 'same' }]), + ], + } as any); + + t.deepEqual(findCredentialIds(project), ['same']); +}); + +test.only('sync-credentials: creates credential yaml file', (t) => { + mock({ '/ws': {} }); + + const project = new Project( + { + id: 'p', + workflows: [baseWorkflow([{ id: 'j', configuration: 'new-id' }])], + } as any, + { credentials: 'credentials.yaml' } + ); + + createProjectCredentials('/ws', project); + + t.true(fs.existsSync('/ws/credentials.yaml')); + const doc = yamlToJson( + fs.readFileSync('/ws/credentials.yaml', 'utf8') + ) as any; + t.deepEqual(doc, { 'new-id': {} }); +}); + +test('sync-credentials: preserves existing credentials and adds missing ones', (t) => { + mock({ + '/ws': {}, + '/ws/credentials.yaml': `existing: + password: secret +`, + }); + + const project = new Project( + { + id: 'p', + workflows: [ + baseWorkflow([ + { id: 'j', configuration: 'existing' }, + { id: 'k', configuration: 'brand-new' }, + ]), + ], + } as any, + { credentials: 'credentials.yaml' } + ); + + createProjectCredentials('/ws', project); + + const doc = yamlToJson( + fs.readFileSync('/ws/credentials.yaml', 'utf8') + ) as any; + t.is(doc.existing.password, 'secret'); + t.deepEqual(doc['brand-new'], {}); +});