Sortby bug

.github/workflows/docs.yml

@@ -2,9 +2,9 @@ on:
   pull_request:
     paths:
       - 'src/**'
-      - 'docs/**'
-      - 'examples/**'
+      - 'documentation/**'
       - 'web/landing/**'
+      - 'Justfile'
       - 'README.md'
   push:
     branches:
@@ -35,7 +35,7 @@ jobs:
           php-version: "8.3"
           dependencies: "locked"
           coverage: "none"
-          extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib'
+          extensions: ':psr, bcmath, curl, dom, hash, json, mbstring, pdo_sqlite, xml, xmlwriter, xmlreader, zip, zlib'
           cache-key-suffix: "-locked"
 
       - name: "Build Docs"

.github/workflows/job-arrow-extension.yml

@@ -32,7 +32,7 @@ jobs:
           install-dependencies: 'false'
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
+        uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
         with:
           toolchain: stable
 
@@ -137,7 +137,7 @@ jobs:
           tools: 'composer:v2, phpize, php-config'
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
+        uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
         with:
           toolchain: stable
 

.github/workflows/job-flow-php-extension.yml

@@ -32,7 +32,7 @@ jobs:
           install-dependencies: 'false'
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
+        uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
         with:
           toolchain: stable
 
@@ -134,7 +134,7 @@ jobs:
           tools: 'composer:v2, phpize, php-config'
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
+        uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
         with:
           toolchain: stable
 

.github/workflows/monorepo-split.yml

@@ -89,6 +89,8 @@ jobs:
             split_repository: 'monolog-telemetry-bridge'
           - local_path: 'src/bridge/openapi/specification'
             split_repository: 'openapi-specification-bridge'
+          - local_path: 'src/bridge/mago/types'
+            split_repository: 'mago-types-bridge'
           - local_path: 'src/bridge/phpstan/types'
             split_repository: 'phpstan-types-bridge'
           - local_path: 'src/bridge/postgresql/valinor'

.github/workflows/test-website.yml

@@ -2,10 +2,14 @@ on:
   pull_request:
     paths:
       - 'web/**'
+      # the playground runs Flow from a phar built out of src, and PlaygroundExamplesTest is the
+      # only thing that executes the published examples
+      - 'src/**'
   push:
     branches: [ 1.x ]
     paths:
       - 'web/**'
+      - 'src/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -15,7 +19,39 @@ permissions:
   contents: read
 
 jobs:
+  build-phar:
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: "Setup PHP Environment"
+        uses: "./.github/actions/setup-php-env"
+        with:
+          php-version: "8.3"
+          dependencies: "locked"
+          coverage: "none"
+          extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib'
+          cache-key-suffix: "-locked-phar"
+          apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev"
+          pie-extensions: "flow-php/pg-query-ext:1.x-dev"
+
+      - name: "Build PHAR file"
+        run: "just phar"
+
+      - name: "Upload Flow PHAR"
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: "website-flow-phar"
+          path: "web/landing/assets/wasm/tools/flow.phar"
+          retention-days: 1
+
   website-tests:
+    needs: build-phar
     runs-on: ${{ matrix.operating-system }}
     strategy:
       fail-fast: false
@@ -53,13 +89,6 @@ jobs:
         with:
           node-version: "22"
 
-      - name: "Setup Chrome"
-        id: chrome
-        uses: browser-actions/setup-chrome@2e1d749697dd1612b833dba4a722266286fbefcd # v2
-        with:
-          chrome-version: "stable"
-          install-chromedriver: true
-
       - name: "Create Wrangler dev.vars"
         run: |
           cat > .dev.vars << EOF
@@ -78,14 +107,28 @@ jobs:
           timeout 30 bash -c 'until curl -s http://localhost:8787 > /dev/null; do sleep 1; done'
         working-directory: "terraform/cloudflare/workers"
 
-      - name: "Test"
-        run: "composer test"
+      # replaces the committed phar so the playground runs this commit's Flow, not whatever was
+      # last checked in
+      - name: "Download Flow PHAR"
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: "website-flow-phar"
+          path: "web/landing/assets/wasm/tools"
+
+      # the website suite runs on the monorepo phpunit, not a copy of its own
+      - name: "Install PHPUnit"
+        run: "composer install --working-dir=tools/phpunit"
+
+      - name: "Install Playwright browsers"
+        run: "vendor/bin/playwright-install --with-deps"
         working-directory: "web/landing"
+
+      - name: "Test"
+        run: "just test-website"
         env:
           PLAYGROUND_API_URL: "http://localhost:8787/api/playground/snippets"
           PLAYGROUND_SNIPPETS_URL: "http://localhost:8787"
-          PANTHER_CHROME_BINARY: ${{ steps.chrome.outputs.chrome-path }}
-          PANTHER_CHROME_DRIVER_BINARY: ${{ steps.chrome.outputs.chromedriver-path }}
+          PLAYWRIGHT_HEADLESS: "true"
 
       - name: "Stop Wrangler"
         if: always()

Justfile

@@ -25,9 +25,13 @@ test *args:
 test-mutation *args:
     tools/infection/vendor/bin/infection --threads=max {{args}}
 
-# Run tests for the landing site.
-test-website:
-    composer test --working-dir=./web/landing
+# Run tests for the landing site, without the published examples.
+test-website *args:
+    tools/phpunit/vendor/bin/phpunit -c web/landing/phpunit.xml --testsuite=unit,integration,functional {{args}}
+
+# Run every published example through the real playground. Minutes, on demand.
+test-examples *args:
+    tools/phpunit/vendor/bin/phpunit -c web/landing/phpunit.xml --testsuite=examples {{args}}
 
 # Run phpbench benchmarks (local only; MUST be inside nix-shell). Args are forwarded to `phpbench run`.
 # Defaults to `--report=flow-report`; pass your own `--report=...` to override it.

benchmarks/src/Datasets/SellersDataset.php

@@ -32,7 +32,7 @@ public function parquet(): string
 
         foreach (data_frame()->read(from_parquet($ordersPath)->withColumns(['seller_id']))->get() as $batch) {
             foreach ($batch->all() as $row) {
-                $sellerId = $row->valueOf('seller_id');
+                $sellerId = $row->get('seller_id');
 
                 if (is_scalar($sellerId) || $sellerId instanceof Stringable) {
                     $sellerIds[(string) $sellerId] = true;

benchmarks/src/Datasets/TextDataset.php

@@ -34,8 +34,8 @@ public function path(): string
 
         foreach (data_frame()->read(from_parquet($parquetPath)->withColumns(['customer', 'notes']))->get() as $batch) {
             foreach ($batch->all() as $row) {
-                $customer = $row->valueOf('customer');
-                $notes = $row->valueOf('notes');
+                $customer = $row->get('customer');
+                $notes = $row->get('notes');
 
                 $customerText = is_scalar($customer) || $customer instanceof Stringable ? (string) $customer : '';
                 $notesText = is_array($notes)

benchmarks/src/Grouping/GroupByAggregateScenario.php

@@ -26,7 +26,7 @@ public function run(): void
         data_frame(BenchmarkConfig::builder())
             ->read(from_floe(Datasets::orders($this->rows)->floe()))
             ->batchSize(1000)
-            ->groupBy(ref('seller_id'))
+            ->groupBy([ref('seller_id')])
             ->aggregate(count(ref('order_id')), sum(ref('discount')), average(ref('discount')), max(ref('discount')))
             ->run();
     }

benchmarks/src/Grouping/GroupByScenario.php

@@ -23,7 +23,7 @@ public function run(): void
         data_frame(BenchmarkConfig::builder())
             ->read(from_floe(Datasets::orders($this->rows)->floe()))
             ->batchSize(1000)
-            ->groupBy(ref('seller_id'))
+            ->groupBy([ref('seller_id')])
             ->aggregate(count(ref('order_id')))
             ->run();
     }

benchmarks/src/Service/Doctrine/DoctrineConnection.php

@@ -40,7 +40,7 @@ public static function createTable(Connection $connection, string $table): void
     {
         $connection
             ->createSchemaManager()
-            ->createTable(to_dbal_schema_table(FakeRandomOrdersExtractor::schema(), $table));
+            ->createTable(to_dbal_schema_table((new FakeRandomOrdersExtractor())->schema(), $table));
     }
 
     public static function dropTable(Connection $connection, string $table): void

benchmarks/src/Service/Postgresql/PostgresqlConnection.php

@@ -42,7 +42,7 @@ public static function open(): Client
 
     public static function createTable(Client $client, string $table): void
     {
-        foreach (to_pgsql_schema_table(FakeRandomOrdersExtractor::schema(), $table)->toSql() as $sql) {
+        foreach (to_pgsql_schema_table((new FakeRandomOrdersExtractor())->schema(), $table)->toSql() as $sql) {
             $client->execute($sql);
         }
     }

benchmarks/src/Service/Seal/SealWriteScenario.php

@@ -49,6 +49,6 @@ public function dropIndex(): void
 
     private function schema(): SealSchema
     {
-        return to_seal_schema(FakeRandomOrdersExtractor::schema(), $this->index(), 'order_id');
+        return to_seal_schema((new FakeRandomOrdersExtractor())->schema(), $this->index(), 'order_id');
     }
 }

benchmarks/src/Sorting/SortOrdersScenario.php

@@ -25,7 +25,7 @@ public function run(): void
         data_frame($config)
             ->read(from_floe(Datasets::orders($this->rows)->floe()))
             ->batchSize(1000)
-            ->sortBy(ref('created_at')->desc())
+            ->sortBy([ref('created_at')->desc()])
             ->run();
     }
 }

benchmarks/src/Transformation/NoopLoader.php

@@ -0,0 +1,14 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\Benchmarks\Transformation;
+
+use Flow\ETL\FlowContext;
+use Flow\ETL\Loader;
+use Flow\ETL\Rows;
+
+final readonly class NoopLoader implements Loader
+{
+    public function load(Rows $rows, FlowContext $context): void {}
+}

benchmarks/src/Transformation/SortByCreatedAt.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Flow\Benchmarks\Transformation;
+
+use Flow\ETL\DataFrame;
+use Flow\ETL\Transformation;
+
+use function Flow\ETL\DSL\ref;
+
+final readonly class SortByCreatedAt implements Transformation
+{
+    public function transform(DataFrame $dataFrame): DataFrame
+    {
+        return $dataFrame->sortBy([ref('created_at')]);
+    }
+}

benchmarks/suites/Transformation/NestedTransformationBench.php

@@ -4,17 +4,12 @@
 
 namespace Flow\Benchmarks\Transformation;
 
-use Flow\ETL\DataFrame;
-use Flow\ETL\FlowContext;
-use Flow\ETL\Rows;
-use Flow\ETL\Transformation;
 use Generator;
 use PhpBench\Attributes as Bench;
 
 use function Flow\ETL\DSL\ref;
 use function Flow\ETL\DSL\select;
 use function Flow\ETL\DSL\to_branch;
-use function Flow\ETL\DSL\to_callable;
 use function Flow\ETL\DSL\to_transformation;
 
 final class NestedTransformationBench
@@ -23,15 +18,7 @@ final class NestedTransformationBench
     #[Bench\Groups(['transformation'])]
     public function bench_blocking_transformation(array $params): void
     {
-        $loader = to_transformation(
-            new class implements Transformation {
-                public function transform(DataFrame $dataFrame): DataFrame
-                {
-                    return $dataFrame->sortBy(ref('created_at'));
-                }
-            },
-            to_callable(static function (Rows $rows, FlowContext $context): void {}),
-        );
+        $loader = to_transformation(new SortByCreatedAt(), new NoopLoader());
 
         (new NestedTransformationScenario((int) $params['rows'], $loader))->run();
     }
@@ -40,15 +27,7 @@ public function transform(DataFrame $dataFrame): DataFrame
     #[Bench\Groups(['transformation'])]
     public function bench_branch_with_transformation(array $params): void
     {
-        $loader = to_branch(
-            ref('order_id')->isNotNull(),
-            to_callable(static function (Rows $rows, FlowContext $context): void {}),
-        )->withTransformation(new class implements Transformation {
-            public function transform(DataFrame $dataFrame): DataFrame
-            {
-                return $dataFrame->sortBy(ref('created_at'));
-            }
-        });
+        $loader = to_branch(ref('order_id')->isNotNull(), new NoopLoader())->withTransformation(new SortByCreatedAt());
 
         (new NestedTransformationScenario((int) $params['rows'], $loader))->run();
     }
@@ -57,10 +36,7 @@ public function transform(DataFrame $dataFrame): DataFrame
     #[Bench\Groups(['transformation'])]
     public function bench_streaming_transformation(array $params): void
     {
-        $loader = to_transformation(
-            select('order_id'),
-            to_callable(static function (Rows $rows, FlowContext $context): void {}),
-        );
+        $loader = to_transformation(select('order_id'), new NoopLoader());
 
         (new NestedTransformationScenario((int) $params['rows'], $loader))->run();
     }

composer.json

@@ -122,6 +122,7 @@
         "flow-php/openapi-specification-bridge": "self.version",
         "flow-php/parquet": "self.version",
         "flow-php/parquet-viewer": "self.version",
+        "flow-php/mago-types-bridge": "self.version",
         "flow-php/phpstan-types-bridge": "self.version",
         "flow-php/postgresql": "self.version",
         "flow-php/postgresql-valinor-bridge": "self.version",
@@ -169,6 +170,7 @@
                 "src/bridge/monolog/http/src/Flow",
                 "src/bridge/monolog/telemetry/src/Flow",
                 "src/bridge/openapi/specification/src/Flow",
+                "src/bridge/mago/types/src/Flow",
                 "src/bridge/phpstan/types/src/Flow",
                 "src/bridge/postgresql/valinor/src/Flow",
                 "src/bridge/psr3/telemetry/src/Flow",
@@ -270,6 +272,7 @@
     },
     "autoload-dev": {
         "psr-4": {
+            "Mago\\": "tools/mago/vendor/carthage-software/mago/composer/src",
             "Flow\\": [
                 "src/adapter/etl-adapter-avro/tests/Flow",
                 "src/adapter/etl-adapter-chartjs/tests/Flow",
@@ -290,6 +293,7 @@
                 "src/bridge/monolog/http/tests/Flow",
                 "src/bridge/monolog/telemetry/tests/Flow",
                 "src/bridge/openapi/specification/tests/Flow",
+                "src/bridge/mago/types/tests/Flow",
                 "src/bridge/phpstan/types/tests/Flow",
                 "src/bridge/postgresql/valinor/tests/Flow",
                 "src/bridge/psr3/telemetry/tests/Flow",

documentation/adrs/schema-immutability.md

@@ -55,7 +55,7 @@ DSL `from_*()` functions stay pure delegation.
 
 ### Out of scope
 
-`EntryReference` remains mutable - `as()`, `asc()` and `desc()` write `$alias` / `$sort` on `$this`. It is shared
+`UnresolvedReference` remains mutable - `as()`, `asc()` and `desc()` write `$alias` / `$sort` on `$this`. It is shared
 with the entire expression DSL, so making it immutable is a separate project and is not attempted here.
 
 ### Breaking change