Integrate ENSDb SDK into ENSApi

.changeset/floppy-years-sneeze.md

@@ -0,0 +1,5 @@
+---
+"@ensnode/ensdb-sdk": minor
+---
+
+Renamed the `client` getter on `EnsDbReader` class to `ensDb`.

.changeset/pink-steaks-think.md

@@ -0,0 +1,5 @@
+---
+"ensapi": minor
+---
+
+Updated custom queries for ENSDb to implement data model from ENSDb SDK.

.changeset/tasty-seals-shout.md

@@ -0,0 +1,5 @@
+---
+"ensapi": minor
+---
+
+Decoupled ENSApi from ENSIndexer by updating the data source for `EnsIndexerPublicConfig` from ENSIndexer to ENSDb.

.changeset/thin-flies-build.md

@@ -0,0 +1,5 @@
+---
+"ensapi": minor
+---
+
+Decoupled ENSApi from ENSIndexer by updating the data source for Indexing Status snapshots, from ENSIndexer's Indexing Status API route to ENSDb.

apps/ensapi/.env.local.example

@@ -2,26 +2,26 @@
 # Optional. If this is not set, the default value is 4334.
 # PORT=4334
 
-# ENSIndexer: The "primary" ENSIndexer service URL.
-# Required. This URL is used to read ENSIndexer's Config and Indexing Status APIs.
-ENSINDEXER_URL=http://localhost:42069
-
 # ENSDb: Database URL
 # Required. This is the connection string for the ENSDb database in which ENSIndexer is storing data.
-# It should match the DATABASE_URL used by the connected ENSIndexer.
+# It should match the DATABASE_URL used by your ENSIndexer.
 # It should be in the format of `postgresql://<username>:<password>@<host>:<port>/<database>`
 #
 # See https://ensnode.io/ensindexer/usage/configuration for additional information.
-# NOTE that ENSApi does NOT need to define DATABASE_SCHEMA, as it is inferred from the connected ENSIndexer's Config.
 DATABASE_URL=postgresql://dbuser:abcd1234@localhost:5432/my_database
 
+# ENSDb: ENSIndexer Schema Name
+# Required. This is a name of the database schema in DATABASE_URL where ENSApi should read indexed data stored by your ENSIndexer.
+# It should match the DATABASE_SCHEMA used by your ENSIndexer.
+ENSINDEXER_SCHEMA_NAME=ensindexer_0
+
 # ENSApi: RPC Configuration
-# Required. ENSApi requires an HTTP RPC to the connected ENSIndexer's ENS Root Chain, which depends
-# on ENSIndexer's NAMESPACE (ex: mainnet, sepolia, ens-test-env). This ENS Root Chain RPC
-# is used to power the Resolution API, in situations where Protocol Acceleration is not possible.
+# Required. ENSApi requires a HTTP RPC exclusively for the ENS Root Chain of the ENS namespace (ex: mainnet, sepolia, ens-test-env) that your ENSIndexer is configured to use.
+# This ENS Root Chain RPC is used to power the Resolution API in situations where Protocol Acceleration is not possible and dynamic RPC calls are required to serve a resolution request.
+# Note that ENS resolution only requires an RPC for the ENS Root Chain. All lookups of data off of the ENS Root Chain are achieved through CCIP-read.
 #
-# When ENSApi starts up it connects to the indicated ENSINDEXER_URL verifies that the ENS Root Chain
-# RPC for the specified namespace is defined.
+# When ENSApi starts up it loads an ENSIndexer public config from ENSDb and verifies that
+# the ENS Root Chain for the configured ENS namespace (ex: mainnet / sepolia / etc) has an RPC defined for ENSApi to use.
 #
 # NOTE: You must configure your own private RPC endpoints. Public RPC endpoints are rate limited and
 # will likely not provide acceptable performance (though this depends on how many non-acceleratable

apps/ensapi/src/cache/indexing-status.cache.ts

@@ -1,48 +1,46 @@
-import config from "@/config";
-
-import {
-  type CrossChainIndexingStatusSnapshot,
-  ENSNodeClient,
-  IndexingStatusResponseCodes,
-  SWRCache,
-} from "@ensnode/ensnode-sdk";
+import { EnsNodeMetadataKeys } from "@ensnode/ensdb-sdk";
+import { type CrossChainIndexingStatusSnapshot, SWRCache } from "@ensnode/ensnode-sdk";
 
+import { ensDbClient } from "@/lib/ensdb/singleton";
 import { makeLogger } from "@/lib/logger";
 
 const logger = makeLogger("indexing-status.cache");
-const client = new ENSNodeClient({ url: config.ensIndexerUrl });
 
 export const indexingStatusCache = new SWRCache<CrossChainIndexingStatusSnapshot>({
   fn: async (_cachedResult) =>
-    client
-      .indexingStatus() // fetch a new indexing status snapshot
-      .then((response) => {
-        if (response.responseCode !== IndexingStatusResponseCodes.Ok) {
-          // An indexing status response was successfully fetched, but the response code contained within the response was not 'ok'.
+    ensDbClient
+      .getIndexingStatusSnapshot() // get the latest indexing status snapshot
+      .then((snapshot) => {
+        if (snapshot === undefined) {
+          // An indexing status snapshot has not been found in ENSDb yet.
+          // This might happen during application startup, i.e. when ENSDb
+          // has not yet been populated with the first snapshot.
           // Therefore, throw an error to trigger the subsequent `.catch` handler.
-          throw new Error("Received Indexing Status response with responseCode other than 'ok'.");
+          throw new Error("Indexing Status snapshot not found in ENSDb yet.");
         }
 
-        logger.info("Fetched Indexing Status to be cached");
-
         // The indexing status snapshot has been fetched and successfully validated for caching.
         // Therefore, return it so that this current invocation of `readCache` will:
         // - Replace the currently cached value (if any) with this new value.
         // - Return this non-null value.
-        return response.realtimeProjection.snapshot;
+        return snapshot;
       })
       .catch((error) => {
-        // Either the indexing status snapshot fetch failed, or the indexing status response was not 'ok'.
+        // Either the indexing status snapshot fetch failed, or the indexing status snapshot was not found in ENSDb yet.
         // Therefore, throw an error so that this current invocation of `readCache` will:
         // - Reject the newly fetched response (if any) such that it won't be cached.
         // - Return the most recently cached value from prior invocations, or `null` if no prior invocation successfully cached a value.
         logger.error(
           error,
-          "Error occurred while fetching a new indexing status snapshot. The cached indexing status snapshot (if any) will not be updated.",
+          `Error occurred while loading Indexing Status snapshot record from ENSNode Metadata table in ENSDb. ` +
+            `Where clause applied: ("ensIndexerSchemaName" = "${ensDbClient.ensIndexerSchemaName}", "key" = "${EnsNodeMetadataKeys.EnsIndexerIndexingStatus}"). ` +
+            `The cached indexing status snapshot (if any) will not be updated.`,
         );
         throw error;
       }),
-  ttl: 5, // 5 seconds
-  proactiveRevalidationInterval: 10, // 10 seconds
+  // We need to refresh the indexing status cache very frequently.
+  // ENSDb won't have issues handling this frequency of queries.
+  ttl: 1, // 1 second
+  proactiveRevalidationInterval: 1, // 1 second
   proactivelyInitialize: true,
 });

apps/ensapi/src/config/config.schema.test.ts

@@ -2,13 +2,15 @@ import packageJson from "@/../package.json" with { type: "json" };
 
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
-import {
-  type ENSIndexerPublicConfig,
-  PluginName,
-  serializeENSIndexerPublicConfig,
-} from "@ensnode/ensnode-sdk";
+import { type ENSIndexerPublicConfig, PluginName } from "@ensnode/ensnode-sdk";
 import type { RpcConfig } from "@ensnode/ensnode-sdk/internal";
 
+vi.mock("@/lib/ensdb/singleton", () => ({
+  ensDbClient: {
+    getEnsIndexerPublicConfig: vi.fn(async () => ENSINDEXER_PUBLIC_CONFIG),
+  },
+}));
+
 import { buildConfigFromEnvironment, buildEnsApiPublicConfig } from "@/config/config.schema";
 import { ENSApi_DEFAULT_PORT } from "@/config/defaults";
 import type { EnsApiEnvironment } from "@/config/environment";
@@ -25,7 +27,6 @@ const VALID_RPC_URL = "https://eth-sepolia.g.alchemy.com/v2/1234";
 
 const BASE_ENV = {
   DATABASE_URL: "postgresql://user:password@localhost:5432/mydb",
-  ENSINDEXER_URL: "http://localhost:42069",
   RPC_URL_1: VALID_RPC_URL,
 } satisfies EnsApiEnvironment;
 
@@ -50,29 +51,16 @@ const ENSINDEXER_PUBLIC_CONFIG = {
   },
 } satisfies ENSIndexerPublicConfig;
 
-const mockFetch = vi.fn();
-vi.stubGlobal("fetch", mockFetch);
-
 describe("buildConfigFromEnvironment", () => {
-  afterEach(() => {
-    mockFetch.mockReset();
-  });
-
   it("returns a valid config object using environment variables", async () => {
-    mockFetch.mockResolvedValueOnce({
-      ok: true,
-      json: () => Promise.resolve(serializeENSIndexerPublicConfig(ENSINDEXER_PUBLIC_CONFIG)),
-    });
-
     await expect(buildConfigFromEnvironment(BASE_ENV)).resolves.toStrictEqual({
       port: ENSApi_DEFAULT_PORT,
       databaseUrl: BASE_ENV.DATABASE_URL,
-      ensIndexerUrl: new URL(BASE_ENV.ENSINDEXER_URL),
       theGraphApiKey: undefined,
 
       ensIndexerPublicConfig: ENSINDEXER_PUBLIC_CONFIG,
       namespace: ENSINDEXER_PUBLIC_CONFIG.namespace,
-      databaseSchemaName: ENSINDEXER_PUBLIC_CONFIG.databaseSchemaName,
+      ensIndexerSchemaName: ENSINDEXER_PUBLIC_CONFIG.databaseSchemaName,
       rpcConfigs: new Map([
         [
           1,
@@ -89,11 +77,6 @@ describe("buildConfigFromEnvironment", () => {
   it("parses CUSTOM_REFERRAL_PROGRAM_EDITIONS as a URL object", async () => {
     const customUrl = "https://example.com/editions.json";
 
-    mockFetch.mockResolvedValueOnce({
-      ok: true,
-      json: () => Promise.resolve(serializeENSIndexerPublicConfig(ENSINDEXER_PUBLIC_CONFIG)),
-    });
-
     const config = await buildConfigFromEnvironment({
       ...BASE_ENV,
       CUSTOM_REFERRAL_PROGRAM_EDITIONS: customUrl,
@@ -116,15 +99,9 @@ describe("buildConfigFromEnvironment", () => {
 
     const TEST_ENV: EnsApiEnvironment = {
       DATABASE_URL: BASE_ENV.DATABASE_URL,
-      ENSINDEXER_URL: BASE_ENV.ENSINDEXER_URL,
     };
 
     it("logs error and exits when CUSTOM_REFERRAL_PROGRAM_EDITIONS is not a valid URL", async () => {
-      mockFetch.mockResolvedValueOnce({
-        ok: true,
-        json: () => Promise.resolve(serializeENSIndexerPublicConfig(ENSINDEXER_PUBLIC_CONFIG)),
-      });
-
       await buildConfigFromEnvironment({
         ...TEST_ENV,
         CUSTOM_REFERRAL_PROGRAM_EDITIONS: "not-a-url",
@@ -137,11 +114,6 @@ describe("buildConfigFromEnvironment", () => {
     });
 
     it("logs error message when QuickNode RPC config was partially configured (missing endpoint name)", async () => {
-      mockFetch.mockResolvedValueOnce({
-        ok: true,
-        json: () => Promise.resolve(serializeENSIndexerPublicConfig(ENSINDEXER_PUBLIC_CONFIG)),
-      });
-
       await buildConfigFromEnvironment({
         ...TEST_ENV,
         QUICKNODE_API_KEY: "my-api-key",
@@ -157,11 +129,6 @@ describe("buildConfigFromEnvironment", () => {
     });
 
     it("logs error message when QuickNode RPC config was partially configured (missing API key)", async () => {
-      mockFetch.mockResolvedValueOnce({
-        ok: true,
-        json: () => Promise.resolve(serializeENSIndexerPublicConfig(ENSINDEXER_PUBLIC_CONFIG)),
-      });
-
       await buildConfigFromEnvironment({
         ...TEST_ENV,
         QUICKNODE_ENDPOINT_NAME: "my-endpoint-name",
@@ -183,10 +150,9 @@ describe("buildEnsApiPublicConfig", () => {
     const mockConfig = {
       port: ENSApi_DEFAULT_PORT,
       databaseUrl: BASE_ENV.DATABASE_URL,
-      ensIndexerUrl: new URL(BASE_ENV.ENSINDEXER_URL),
       ensIndexerPublicConfig: ENSINDEXER_PUBLIC_CONFIG,
       namespace: ENSINDEXER_PUBLIC_CONFIG.namespace,
-      databaseSchemaName: ENSINDEXER_PUBLIC_CONFIG.databaseSchemaName,
+      ensIndexerSchemaName: ENSINDEXER_PUBLIC_CONFIG.databaseSchemaName,
       rpcConfigs: new Map([
         [
           1,
@@ -215,10 +181,9 @@ describe("buildEnsApiPublicConfig", () => {
     const mockConfig = {
       port: ENSApi_DEFAULT_PORT,
       databaseUrl: BASE_ENV.DATABASE_URL,
-      ensIndexerUrl: new URL(BASE_ENV.ENSINDEXER_URL),
       ensIndexerPublicConfig: ENSINDEXER_PUBLIC_CONFIG,
       namespace: ENSINDEXER_PUBLIC_CONFIG.namespace,
-      databaseSchemaName: ENSINDEXER_PUBLIC_CONFIG.databaseSchemaName,
+      ensIndexerSchemaName: ENSINDEXER_PUBLIC_CONFIG.databaseSchemaName,
       rpcConfigs: new Map(),
       customReferralProgramEditionConfigSetUrl: undefined,
     };
@@ -245,14 +210,13 @@ describe("buildEnsApiPublicConfig", () => {
     const mockConfig = {
       port: ENSApi_DEFAULT_PORT,
       databaseUrl: BASE_ENV.DATABASE_URL,
-      ensIndexerUrl: new URL(BASE_ENV.ENSINDEXER_URL),
       ensIndexerPublicConfig: {
         ...ENSINDEXER_PUBLIC_CONFIG,
         plugins: ["subgraph"],
         isSubgraphCompatible: true,
       },
       namespace: ENSINDEXER_PUBLIC_CONFIG.namespace,
-      databaseSchemaName: ENSINDEXER_PUBLIC_CONFIG.databaseSchemaName,
+      ensIndexerSchemaName: ENSINDEXER_PUBLIC_CONFIG.databaseSchemaName,
       rpcConfigs: new Map(),
       customReferralProgramEditionConfigSetUrl: undefined,
       theGraphApiKey: "secret-api-key",

apps/ensapi/src/config/config.schema.ts

@@ -1,16 +1,13 @@
 import packageJson from "@/../package.json" with { type: "json" };
 
 import pRetry from "p-retry";
-import { parse as parseConnectionString } from "pg-connection-string";
 import { prettifyError, ZodError, z } from "zod/v4";
 
 import type { EnsApiPublicConfig } from "@ensnode/ensnode-sdk";
 import {
   buildRpcConfigsFromEnv,
   canFallbackToTheGraph,
-  DatabaseSchemaNameSchema,
   ENSNamespaceSchema,
-  EnsIndexerUrlSchema,
   invariant_rpcConfigsSpecifiedForRootChain,
   makeENSIndexerPublicConfigSchema,
   OptionalPortNumberSchema,
@@ -19,29 +16,12 @@ import {
 } from "@ensnode/ensnode-sdk/internal";
 
 import { ENSApi_DEFAULT_PORT } from "@/config/defaults";
+import { EnsDbConfigSchema } from "@/config/ensdb-config.schema";
 import type { EnsApiEnvironment } from "@/config/environment";
 import { invariant_ensIndexerPublicConfigVersionInfo } from "@/config/validations";
-import { fetchENSIndexerConfig } from "@/lib/fetch-ensindexer-config";
+import { ensDbClient } from "@/lib/ensdb/singleton";
 import logger from "@/lib/logger";
 
-export const DatabaseUrlSchema = z.string().refine(
-  (url) => {
-    try {
-      if (!url.startsWith("postgresql://") && !url.startsWith("postgres://")) {
-        return false;
-      }
-      const config = parseConnectionString(url);
-      return !!(config.host && config.port && config.database);
-    } catch {
-      return false;
-    }
-  },
-  {
-    error:
-      "Invalid PostgreSQL connection string. Expected format: postgresql://username:password@host:port/database",
-  },
-);
-
 /**
  * Schema for validating custom referral program edition config set URL.
  */
@@ -63,15 +43,13 @@ const CustomReferralProgramEditionConfigSetUrlSchema = z
 const EnsApiConfigSchema = z
   .object({
     port: OptionalPortNumberSchema.default(ENSApi_DEFAULT_PORT),
-    databaseUrl: DatabaseUrlSchema,
-    databaseSchemaName: DatabaseSchemaNameSchema,
-    ensIndexerUrl: EnsIndexerUrlSchema,
     theGraphApiKey: TheGraphApiKeySchema,
     namespace: ENSNamespaceSchema,
     rpcConfigs: RpcConfigsSchema,
     ensIndexerPublicConfig: makeENSIndexerPublicConfigSchema("ensIndexerPublicConfig"),
     customReferralProgramEditionConfigSetUrl: CustomReferralProgramEditionConfigSetUrlSchema,
   })
+  .extend(EnsDbConfigSchema.shape)
   .check(invariant_rpcConfigsSpecifiedForRootChain)
   .check(invariant_ensIndexerPublicConfigVersionInfo);
 
@@ -85,27 +63,38 @@ export type EnsApiConfig = z.infer<typeof EnsApiConfigSchema>;
  */
 export async function buildConfigFromEnvironment(env: EnsApiEnvironment): Promise<EnsApiConfig> {
   try {
-    const ensIndexerUrl = EnsIndexerUrlSchema.parse(env.ENSINDEXER_URL);
+    // TODO: transfer the responsibility of fetching
+    // the ENSIndexer Public Config to a middleware layer, as per:
+    // https://github.com/namehash/ensnode/issues/1806
+    const ensIndexerPublicConfig = await pRetry(
+      async () => {
+        const config = await ensDbClient.getEnsIndexerPublicConfig();
 
-    const ensIndexerPublicConfig = await pRetry(() => fetchENSIndexerConfig(ensIndexerUrl), {
-      retries: 3,
-      onFailedAttempt: ({ error, attemptNumber, retriesLeft }) => {
-        logger.info(
-          `ENSIndexer Config fetch attempt ${attemptNumber} failed (${error.message}). ${retriesLeft} retries left.`,
-        );
+        if (!config) {
+          throw new Error("ENSIndexer Public Config not yet available in ENSDb.");
+        }
+
+        return config;
       },
-    });
+      {
+        retries: 13, // This allows for a total of over 1 hour of retries with the exponential backoff strategy
+        onFailedAttempt: ({ error, attemptNumber, retriesLeft }) => {
+          logger.info(
+            `ENSIndexer Public Config fetch attempt ${attemptNumber} failed (${error.message}). ${retriesLeft} retries left.`,
+          );
+        },
+      },
+    );
 
     const rpcConfigs = buildRpcConfigsFromEnv(env, ensIndexerPublicConfig.namespace);
 
     return EnsApiConfigSchema.parse({
       port: env.PORT,
       databaseUrl: env.DATABASE_URL,
-      ensIndexerUrl: env.ENSINDEXER_URL,
       theGraphApiKey: env.THEGRAPH_API_KEY,
       ensIndexerPublicConfig,
       namespace: ensIndexerPublicConfig.namespace,
-      databaseSchemaName: ensIndexerPublicConfig.databaseSchemaName,
+      ensIndexerSchemaName: ensIndexerPublicConfig.databaseSchemaName,
       rpcConfigs,
       customReferralProgramEditionConfigSetUrl: env.CUSTOM_REFERRAL_PROGRAM_EDITIONS,
     });