feat: auto create credentials.yaml

.changeset/fine-seas-divide.md

@@ -0,0 +1,7 @@
+---
+'@openfn/engine-multi': minor
+'@openfn/runtime': minor
+'@openfn/cli': minor
+---
+
+Auto create credentials.yaml for projects

.changeset/odd-fans-taste.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Auto generates credentials.yaml for pulled projects

packages/cli/src/execute/handler.ts

@@ -1,6 +1,4 @@
 import type { ExecutionPlan } from '@openfn/lexicon';
-import { yamlToJson } from '@openfn/project';
-import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 
 import type { ExecuteOptions } from './command';
@@ -22,6 +20,7 @@ import fuzzyMatchStep from '../util/fuzzy-match-step';
 import abort from '../util/abort';
 import validatePlan from '../util/validate-plan';
 import overridePlanAdaptors from '../util/override-plan-adaptors';
+import { loadCredentialMap } from '../util/load-credential-map';
 
 const matchStep = (
   plan: ExecutionPlan,
@@ -56,15 +55,9 @@ const loadAndApplyCredentialMap = async (
   let creds = {};
   if (options.credentials) {
     try {
-      const credsRaw = await readFile(
-        path.resolve(options.workspace!, options.credentials),
-        'utf8'
+      creds = loadCredentialMap(
+        path.resolve(options.workspace!, options.credentials)
       );
-      if (options.credentials.endsWith('.json')) {
-        creds = JSON.parse(credsRaw);
-      } else {
-        creds = yamlToJson(credsRaw);
-      }
       logger.info('Credential map loaded ');
     } catch (e: any) {
       // If we get here, the credential map failed to load

packages/cli/src/projects/checkout.ts

@@ -15,6 +15,7 @@ import {
   tidyWorkflowDir,
   updateForkedFrom,
 } from './util';
+import { createProjectCredentials } from './create-credentials';
 
 export type CheckoutOptions = Pick<
   Opts,
@@ -125,5 +126,8 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => {
       logger?.warn('WARNING! No content for file', f);
     }
   }
+
+  createProjectCredentials(workspacePath, switchProject, logger);
+
   logger?.success(`Expanded project to ${workspacePath}`);
 };

packages/cli/src/projects/create-credentials.ts

@@ -0,0 +1,63 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import type Project from '@openfn/project';
+import { jsonToYaml } from '@openfn/project';
+
+import { loadCredentialMap } from '../util/load-credential-map';
+import type { Logger } from '../util/logger';
+
+export function findCredentialIds(project: Project): string[] {
+  const ids = new Set<string>();
+  for (const wf of project.workflows) {
+    for (const step of wf.steps) {
+      const job = step as { configuration?: string | null };
+      const { configuration } = job;
+      if (
+        typeof configuration === 'string' &&
+        configuration &&
+        !configuration.endsWith('.json')
+      ) {
+        ids.add(configuration);
+      }
+    }
+  }
+  return Array.from(ids);
+}
+
+export function createProjectCredentials(
+  workspacePath: string,
+  project: Project,
+  logger?: Logger
+): void {
+  const credentialsPath = project.config.credentials;
+  if (typeof credentialsPath !== 'string') return;
+
+  const ids = findCredentialIds(project);
+  if (!ids.length) return;
+
+  const absolutePath = path.resolve(workspacePath, credentialsPath);
+  let existing: Record<string, unknown> = {};
+
+  try {
+    existing = loadCredentialMap(absolutePath);
+  } catch (e: any) {
+    // project doesn't have credential
+  }
+
+  const new_creds = ids.filter((id) => !(id in existing)).sort();
+  if (!new_creds.length) return;
+
+  const merged: Record<string, unknown> = { ...existing };
+  for (const id of new_creds) {
+    merged[id] = {};
+  }
+
+  const content = credentialsPath.endsWith('.json')
+    ? `${JSON.stringify(merged, null, 2)}`
+    : jsonToYaml(merged);
+
+  fs.mkdirSync(path.dirname(absolutePath), { recursive: true });
+  fs.writeFileSync(absolutePath, content, 'utf8');
+  logger?.debug(`Added ${new_creds.length} credentials to ${credentialsPath}`);
+}

packages/cli/src/util/load-credential-map.ts

@@ -0,0 +1,23 @@
+import fs from 'node:fs';
+import { yamlToJson } from '@openfn/project';
+
+export function loadCredentialMap(filePath: string): Record<string, unknown> {
+  const raw = fs.readFileSync(filePath, 'utf8');
+  if (!raw.trim()) return {};
+
+  if (filePath.endsWith('.json')) {
+    const parsed = JSON.parse(raw) as unknown;
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      throw new Error('credential file contains invalid JSON');
+    }
+    return parsed as Record<string, unknown>;
+  } else {
+    const parsed = yamlToJson(raw) as unknown;
+    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+      return parsed as Record<string, unknown>;
+    } else if (parsed != null) {
+      throw new Error('credential file contains invalid YAML');
+    }
+    return {};
+  }
+}

packages/cli/test/projects/create-credentials.test.ts

@@ -0,0 +1,102 @@
+import test from 'ava';
+import fs from 'node:fs';
+import mock from 'mock-fs';
+
+import Project from '@openfn/project';
+import { yamlToJson } from '@openfn/project';
+
+import {
+  findCredentialIds,
+  createProjectCredentials,
+} from '../../src/projects/create-credentials';
+
+test.afterEach(() => {
+  try {
+    mock.restore();
+  } catch {}
+});
+
+const baseWorkflow = (steps: any[]) => ({
+  id: 'wf',
+  name: 'wf',
+  history: [],
+  steps,
+});
+
+test('sync-credentials: inline string references', (t) => {
+  const project = new Project({
+    id: 'p',
+    workflows: [
+      baseWorkflow([
+        { id: 'a', configuration: 'owner|cred' },
+        { id: 'c', configuration: 'ignored.json' },
+        { id: 'd', configuration: '' },
+      ]),
+    ],
+  } as any);
+
+  const ids = findCredentialIds(project);
+  t.deepEqual(ids, ['owner|cred']);
+});
+
+test('sync-credentials: ignores duplicate references', (t) => {
+  const project = new Project({
+    id: 'p',
+    workflows: [
+      baseWorkflow([{ id: 'a', configuration: 'same' }]),
+      baseWorkflow([{ id: 'b', configuration: 'same' }]),
+    ],
+  } as any);
+
+  t.deepEqual(findCredentialIds(project), ['same']);
+});
+
+test.only('sync-credentials: creates credential yaml file', (t) => {
+  mock({ '/ws': {} });
+
+  const project = new Project(
+    {
+      id: 'p',
+      workflows: [baseWorkflow([{ id: 'j', configuration: 'new-id' }])],
+    } as any,
+    { credentials: 'credentials.yaml' }
+  );
+
+  createProjectCredentials('/ws', project);
+
+  t.true(fs.existsSync('/ws/credentials.yaml'));
+  const doc = yamlToJson(
+    fs.readFileSync('/ws/credentials.yaml', 'utf8')
+  ) as any;
+  t.deepEqual(doc, { 'new-id': {} });
+});
+
+test('sync-credentials: preserves existing credentials and adds missing ones', (t) => {
+  mock({
+    '/ws': {},
+    '/ws/credentials.yaml': `existing:
+  password: secret
+`,
+  });
+
+  const project = new Project(
+    {
+      id: 'p',
+      workflows: [
+        baseWorkflow([
+          { id: 'j', configuration: 'existing' },
+          { id: 'k', configuration: 'brand-new' },
+        ]),
+      ],
+    } as any,
+    { credentials: 'credentials.yaml' }
+  );
+
+  createProjectCredentials('/ws', project);
+
+  const doc = yamlToJson(
+    fs.readFileSync('/ws/credentials.yaml', 'utf8')
+  ) as any;
+  t.is(doc.existing.password, 'secret');
+  t.deepEqual(doc['brand-new'], {});
+});