feat(search): add list fields endpoint + pagination

.github/workflows/ci.yml

@@ -55,16 +55,24 @@ jobs:
       - name: Cleanup Disk Space
         run: |
           df -h
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /usr/share/swift
-          sudo rm -rf /usr/local/.ghcup
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
-          df -h
+
+          if [ "$(df -BG / | awk 'NR==2 {gsub("G","",$4); print $4}')" -lt 30 ]; then
+            echo "Less than 30GiB available. Running cleanup..."
+            sudo rm -rf /usr/share/dotnet
+            sudo rm -rf /usr/local/lib/android
+            sudo rm -rf /usr/share/swift
+            sudo rm -rf /usr/local/.ghcup
+            sudo rm -rf /opt/hostedtoolcache/CodeQL
+            df -h
+          else
+            echo "30GiB or more available. Skipping cleanup."
+          fi
 
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
       - name: Install Ubuntu packages
-        run: sudo apt-get -y install protobuf-compiler
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install protobuf-compiler
       - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v.6.1.0
         with:
           python-version: '3.11'
@@ -88,9 +96,10 @@ jobs:
         if: steps.modified.outputs.rust_src == 'true'
         with:
           workspaces: "./quickwit -> target"
+          shared-key: "quickwit-cargo"
       - name: Install nextest
         if: always() && steps.modified.outputs.rust_src == 'true'
-        uses: taiki-e/cache-cargo-install-action@34ce5120836e5f9f1508d8713d7fdea0e8facd6f # v3.0.1
+        uses: taiki-e/install-action@aba36d755ec7ca22d38b12111787c26115943952
         with:
           tool: cargo-nextest
       - name: cargo build
@@ -132,7 +141,9 @@ jobs:
               - .github/workflows/ci.yml
       - name: Install Ubuntu packages
         if: always() && steps.modified.outputs.rust_src == 'true'
-        run: sudo apt-get -y install protobuf-compiler
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install protobuf-compiler
       - name: Setup nightly Rust Toolchain (for rustfmt)
         if: steps.modified.outputs.rust_src == 'true'
         uses: dtolnay/rust-toolchain@f7ccc83f9ed1e5b9c81d8a67d7ad1a747e22a561 # master
@@ -149,6 +160,7 @@ jobs:
         uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2
         with:
           workspaces: "./quickwit -> target"
+          shared-key: "quickwit-cargo"
       - name: Install cargo deny
         if: always() && steps.modified.outputs.rust_src == 'true'
         uses: taiki-e/cache-cargo-install-action@34ce5120836e5f9f1508d8713d7fdea0e8facd6f # v3.0.1

.github/workflows/ui-ci.yml

@@ -18,32 +18,36 @@ permissions:
   contents: read
 
 jobs:
-  tests:
-    name: ${{ matrix.task.name }}
+  checks:
+    name: Lint, type check & unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
+        with:
+          node-version: 24
+          cache: "yarn"
+          cache-dependency-path: quickwit/quickwit-ui/yarn.lock
+      - name: Install JS dependencies
+        run: yarn --cwd quickwit-ui install
+        working-directory: ./quickwit
+      - name: Lint
+        run: yarn --cwd quickwit-ui lint
+        working-directory: ./quickwit
+      - name: Type check
+        run: yarn --cwd quickwit-ui type
+        working-directory: ./quickwit
+      - name: Unit tests
+        run: yarn --cwd quickwit-ui test
+        working-directory: ./quickwit
+
+  e2e:
+    name: Playwright e2e
     runs-on: ubuntu-latest
     permissions:
       contents: read
       actions: write
-    strategy:
-      fail-fast: false
-      matrix:
-        task:
-          - name: Cypress run
-            command: |
-              sudo apt-get -y install protobuf-compiler
-              CI=false yarn --cwd quickwit-ui build
-              RUSTFLAGS="--cfg tokio_unstable" cargo build --features=postgres
-              mkdir qwdata
-              RUSTFLAGS="--cfg tokio_unstable" cargo run --features=postgres -- run --service searcher --service metastore --config ../config/quickwit.yaml &
-              yarn --cwd quickwit-ui cypress run
-          - name: Lint
-            command: yarn --cwd quickwit-ui lint
-          - name: Check type consistency
-            command: yarn --cwd quickwit-ui type
-          - name: Unit Test
-            command: yarn --cwd quickwit-ui test
     services:
-      # PostgreSQL service container
       postgres:
         image: postgres:latest
         ports:
@@ -52,7 +56,6 @@ jobs:
           POSTGRES_USER: quickwit-dev
           POSTGRES_PASSWORD: quickwit-dev
           POSTGRES_DB: quickwit-metastore-dev
-        # Set health checks to wait until postgres has started
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
@@ -61,7 +64,7 @@ jobs:
     env:
       CARGO_INCREMENTAL: 0
       RUST_BACKTRACE: 1
-      RUSTFLAGS: -Dwarnings -C lto=off
+      RUSTFLAGS: -Dwarnings --cfg tokio_unstable
       RUSTDOCFLAGS: -Dwarnings -Arustdoc::private_intra_doc_links
       QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@postgres:5432/quickwit-metastore-dev
     steps:
@@ -75,14 +78,28 @@ jobs:
         uses: dtolnay/rust-toolchain@f7ccc83f9ed1e5b9c81d8a67d7ad1a747e22a561 # master
         with:
           toolchain: stable
-      - name: Install JS dependencies
-        run: yarn --cwd quickwit-ui install
-        working-directory: ./quickwit
       - name: Setup Rust cache
-        if: matrix.task.name == 'Cypress run'
         uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2
         with:
           workspaces: "./quickwit -> target"
-      - name: ${{ matrix.task.name }}
-        run: ${{ matrix.task.command }}
+          shared-key: "quickwit-cargo"
+      - name: Install JS dependencies
+        run: yarn --cwd quickwit-ui install
+        working-directory: ./quickwit
+      - name: Install Playwright browsers
+        run: npx playwright install chromium --with-deps --only-shell
+        working-directory: ./quickwit/quickwit-ui
+      - name: Build UI
+        run: CI=false yarn --cwd quickwit-ui build
+        working-directory: ./quickwit
+      - name: Build Quickwit
+        run: |
+          sudo apt-get update && sudo apt-get -y install protobuf-compiler
+          cargo build --features=postgres
+        working-directory: ./quickwit
+      - name: Run e2e tests
+        run: |
+          mkdir -p qwdata
+          cargo run --features=postgres -- run --service searcher --service metastore --config ../config/quickwit.yaml &
+          yarn --cwd quickwit-ui e2e-test
         working-directory: ./quickwit

LICENSE-3rdparty.csv

@@ -281,6 +281,7 @@ oneshot,https://github.com/faern/oneshot,MIT OR Apache-2.0,Linus Färnstrand <fa
 oorandom,https://hg.sr.ht/~icefox/oorandom,MIT,Simon Heath <icefox@dreamquest.io>
 openssl-probe,https://github.com/alexcrichton/openssl-probe,MIT OR Apache-2.0,Alex Crichton <alex@alexcrichton.com>
 opentelemetry,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry,Apache-2.0,The opentelemetry Authors
+opentelemetry-appender-tracing,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-appender-tracing,Apache-2.0,The opentelemetry-appender-tracing Authors
 opentelemetry-http,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-http,Apache-2.0,The opentelemetry-http Authors
 opentelemetry-otlp,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-otlp,Apache-2.0,The opentelemetry-otlp Authors
 opentelemetry-proto,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-proto,Apache-2.0,The opentelemetry-proto Authors

docs/reference/es_compatible_api.md

@@ -365,6 +365,79 @@ Example response:
 
 [HTTP accept header]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
 
+
+### `_field_caps`   Field capabilities API
+
+```
+GET api/v1/_elastic/<index>/_field_caps
+```
+```
+POST api/v1/_elastic/<index>/_field_caps
+```
+```
+GET api/v1/_elastic/_field_caps
+```
+```
+POST api/v1/_elastic/_field_caps
+```
+
+The [field capabilities API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-field-caps.html) returns information about the capabilities of fields among multiple indices.
+
+#### Supported Query string parameters
+
+| Variable              | Type       | Description                                                                    | Default value |
+| --------------------- | ---------- | ------------------------------------------------------------------------------ | ------------- |
+| `fields`              | `String`   | Comma-separated list of fields to retrieve capabilities for. Supports wildcards (`*`). | (Optional) |
+| `allow_no_indices`    | `Boolean`  | If `true`, missing or closed indices are not an error.                          | (Optional)    |
+| `expand_wildcards`    | `String`   | Controls what kind of indices that wildcard patterns can match.                 | (Optional)    |
+| `ignore_unavailable`  | `Boolean`  | If `true`, unavailable indices are ignored.                                    | (Optional)    |
+| `start_timestamp`     | `Integer`  | *(Quickwit-specific)* If set, restricts splits to documents with a timestamp range start >= `start_timestamp` (seconds since epoch). | (Optional) |
+| `end_timestamp`       | `Integer`  | *(Quickwit-specific)* If set, restricts splits to documents with a timestamp range end < `end_timestamp` (seconds since epoch). | (Optional) |
+
+#### Supported Request Body parameters
+
+| Variable           | Type          | Description                                                                 | Default value |
+| ------------------ | ------------- | --------------------------------------------------------------------------- | ------------- |
+| `index_filter`     | `Json object` | A query to filter indices. If provided, only fields from indices that can potentially match the filter are returned. See [index_filter](#index_filter). | (Optional) |
+| `runtime_mappings`  | `Json object` | Accepted but not supported.                                                 | (Optional)    |
+
+#### `index_filter`
+
+The `index_filter` parameter allows you to filter which indices contribute to the field capabilities response. When provided, Quickwit uses the filter query to prune indices (splits) that cannot match the filter, and only returns field capabilities for the remaining ones.
+
+Like Elasticsearch, this is a **best-effort** approach: Quickwit may return field capabilities from indices that do not actually contain any matching documents. In Quickwit, the filtering is limited to the existing split-pruning based on metadata:
+
+- **Time pruning**: Range queries on the timestamp field can eliminate splits whose time range does not overlap with the filter.
+- **Tag pruning**: Term queries on [tag fields](../configuration/index-config.md#tag-fields) can eliminate splits that do not contain the requested tag value.
+
+Other filter types (e.g. full-text queries or term queries on non-tag fields) are accepted but will not prune any splits — all indices will be returned as if no filter was specified. In particular, Quickwit does not check whether terms are present in the term dictionary.
+
+#### Request Body example
+
+```json
+{
+  "index_filter": {
+    "range": {
+      "timestamp": {
+        "gte": "2024-01-01T00:00:00Z",
+        "lt": "2024-02-01T00:00:00Z"
+      }
+    }
+  }
+}
+```
+
+```json
+{
+  "index_filter": {
+    "term": {
+      "status": "active"
+    }
+  }
+}
+```
+
+
 ## Query DSL
 
 [Elasticsearch Query DSL reference](https://www.elastic.co/guide/en/elasticsearch/reference/8.8/query-dsl.html).

docs/reference/metrics.md

@@ -42,8 +42,8 @@ Currently Quickwit exposes metrics for three caches: `fastfields`, `shortlived`,
 
 | Namespace | Metric Name | Description | Type |
 | --------- | ----------- | ----------- | ---- |
-| `quickwit_ingest` | `ingested_num_bytes` | Total size of the docs ingested in bytes | `counter` |
-| `quickwit_ingest` | `ingested_num_docs` | Number of docs received to be ingested | `counter` |
+| `quickwit_ingest` | `docs_bytes_total` | Total size of the docs ingested, measured in ingester's leader, after validation and before persistence/replication | `counter` |
+| `quickwit_ingest` | `docs_total` | Total number of the docs ingested, measured in ingester's leader, after validation and before persistence/replication | `counter` |
 | `quickwit_ingest` | `queue_count` | Number of queues currently active | `counter` |
 
 ## Metastore Metrics

quickwit/Cargo.toml

@@ -164,6 +164,7 @@ oneshot = "0.1"
 openssl = { version = "0.10", default-features = false }
 openssl-probe = "0.1"
 opentelemetry = "0.31"
+opentelemetry-appender-tracing = "0.31"
 opentelemetry_sdk = { version = "0.31", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic"] }
 ouroboros = "0.18"

quickwit/quickwit-cli/Cargo.toml

@@ -37,6 +37,7 @@ numfmt = { workspace = true }
 once_cell = { workspace = true }
 openssl-probe = { workspace = true, optional = true }
 opentelemetry = { workspace = true }
+opentelemetry-appender-tracing = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 opentelemetry-otlp = { workspace = true }
 reqwest = { workspace = true }
@@ -101,6 +102,7 @@ release-feature-set = [
   "quickwit-indexing/pulsar",
   "quickwit-indexing/sqs",
   "quickwit-indexing/vrl",
+  "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
   "quickwit-metastore/postgres",
@@ -114,6 +116,7 @@ release-feature-vendored-set = [
   "quickwit-indexing/sqs",
   "quickwit-indexing/vrl",
   "quickwit-indexing/vendored-kafka",
+  "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
   "quickwit-metastore/postgres",
@@ -126,6 +129,7 @@ release-macos-feature-vendored-set = [
   "quickwit-indexing/sqs",
   "quickwit-indexing/vrl",
   "quickwit-indexing/vendored-kafka-macos",
+  "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
   "quickwit-metastore/postgres",