Move away from row based memory layout towards columns

.github/actions/pg-query-ext/action.yml

@@ -0,0 +1,41 @@
+name: "pg-query-ext from source"
+description: "Builds src/extension/pg-query-ext from the checked-out source, cached by its source hash, and enables it"
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Resolve PHP version"
+      id: php
+      shell: bash
+      run: echo "version=$(php -r 'echo PHP_VERSION;')" >> "$GITHUB_OUTPUT"
+
+    - name: "Cache pg_query.so"
+      id: cache
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+      with:
+        path: src/extension/pg-query-ext/ext/modules/pg_query.so
+        key: pg-query-ext-${{ runner.os }}-${{ runner.arch }}-php-${{ steps.php.outputs.version }}-${{ hashFiles('src/extension/pg-query-ext/**') }}
+
+    - name: "Install build dependencies"
+      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      shell: bash
+      run: sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev
+
+    - name: "Build pg-query-ext"
+      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      shell: bash
+      working-directory: src/extension/pg-query-ext
+      # a shallow checkout has no tags for config.m4 to describe; this build only runs the test suite
+      run: |
+        if ! git describe --tags --match '[0-9]*' >/dev/null 2>&1; then
+          export PG_QUERY_EXT_VERSION="0.0.0-0-g$(git rev-parse --short HEAD)"
+        fi
+        make build
+
+    - name: "Enable pg-query-ext"
+      shell: bash
+      working-directory: src/extension/pg-query-ext
+      run: |
+        sudo cp ext/modules/pg_query.so "$(php -r 'echo ini_get("extension_dir");')/"
+        echo "extension=pg_query.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')"
+        php -m | grep -x pg_query

.github/workflows/baseline.yml

@@ -46,7 +46,9 @@ jobs:
           extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib'
           cache-key-suffix: "-locked-phar"
           apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev"
-          pie-extensions: "flow-php/pg-query-ext:1.x-dev"
+
+      - name: "Build pg-query-ext from source"
+        uses: "./.github/actions/pg-query-ext"
 
       - name: "Build PHAR file"
         run: "just phar"
@@ -116,11 +118,13 @@ jobs:
           extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib'
           cache-key-suffix: "-website-phar"
           apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev"
-          pie-extensions: "flow-php/pg-query-ext:1.x-dev"
           extra-working-directory: "web/landing"
           extra-cache-key-suffix: "-website"
           extra-composer-file: "web/landing/composer.lock"
 
+      - name: "Build pg-query-ext from source"
+        uses: "./.github/actions/pg-query-ext"
+
       - name: "Generate documentation"
         run: "just docs"
 

.github/workflows/job-arrow-extension.yml

@@ -34,7 +34,7 @@ jobs:
           install-dependencies: 'false'
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
+        uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable
         with:
           toolchain: stable
 
@@ -86,6 +86,12 @@ jobs:
         working-directory: src/extension/arrow-ext
         run: make build
 
+      - name: Install Composer Dependencies
+        uses: ./.github/actions/composer-install
+        with:
+          php-version: ${{ matrix.php }}
+          dependencies: locked
+
       - name: Run PHPT tests
         working-directory: src/extension/arrow-ext
         run: make test
@@ -98,12 +104,6 @@ jobs:
           echo "extension=arrow.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')"
           php -m | grep arrow
 
-      - name: Install Composer Dependencies
-        uses: ./.github/actions/composer-install
-        with:
-          php-version: ${{ matrix.php }}
-          dependencies: locked
-
       - name: Run Parquet Integration Tests with Arrow
         run: just test --testsuite=lib-parquet-integration ${{ (matrix.php == '8.3' && runner.os == 'Linux') && '--coverage-clover=./var/phpunit/coverage/clover/coverage.xml' || '' }}
 
@@ -141,7 +141,7 @@ jobs:
           tools: 'composer:v2, phpize, php-config'
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
+        uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable
         with:
           toolchain: stable
 
@@ -202,7 +202,13 @@ jobs:
 
       - name: Verify extension version
         run: |
-          expected="$(git describe --tags --match '[0-9]*' | sed -E 's/-([0-9]+)-(g[0-9a-f]+)$/+\1.\2/')"
+          describe="$(git describe --tags --match '[0-9]*')"
+          if git describe --exact-match --tags --match '[0-9]*' HEAD >/dev/null 2>&1; then
+            expected="$describe"
+          else
+            # commits past a tag are a pre-release of the next minor: 0.44.1-65-g4525c4adc -> 0.45.0-dev+65.g4525c4adc
+            expected="$(echo "$describe" | awk -F- '{split($1, v, "."); printf "%d.%d.0-dev+%s.%s", v[1], v[2] + 1, $2, $3}')"
+          fi
           actual="$(php -r 'echo phpversion("arrow");')"
           echo "expected: $expected, actual: $actual"
           test "$actual" = "$expected"

.github/workflows/job-flow-php-extension.yml

@@ -34,7 +34,7 @@ jobs:
           install-dependencies: 'false'
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
+        uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable
         with:
           toolchain: stable
 
@@ -45,7 +45,8 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
             src/extension/flow-php-ext/target
-          key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/flow-php-ext/Cargo.lock') }}
+            src/extension/arrow-ext/target
+          key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/flow-php-ext/Cargo.lock', 'src/extension/arrow-ext/Cargo.lock') }}
           restore-keys: |
             ${{ runner.os }}-php${{ matrix.php }}-cargo-
 
@@ -76,10 +77,29 @@ jobs:
           echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV"
           echo "Detected libclang at: $LIBCLANG_SO"
 
+      - name: Verify the nix development version literals
+        run: |
+          if git describe --exact-match --tags --match '[0-9]*' HEAD >/dev/null 2>&1; then
+            exit 0
+          fi
+          tag="$(git describe --tags --abbrev=0 --match '[0-9]*')"
+          expected="$(echo "$tag" | awk -F. '{printf "%d.%d.0-dev", $1, $2 + 1}')"
+          for f in .nix/pkgs/php-flow-php-ext/package.nix .nix/pkgs/php-arrow-ext/package.nix; do
+            grep -q "version ? \"$expected\"" "$f" || { echo "$f: version literal must be $expected"; exit 1; }
+          done
+
+      - name: Build arrow extension
+        working-directory: src/extension/arrow-ext
+        run: make build
+
       - name: Build extension
         working-directory: src/extension/flow-php-ext
         run: make build
 
+      - name: Test flow-batch-frame crate
+        working-directory: src/extension/flow-php-ext/crates/flow-batch-frame
+        run: cargo test --locked
+
       - name: Install Composer Dependencies
         uses: ./.github/actions/composer-install
         with:
@@ -88,7 +108,7 @@ jobs:
 
       - name: Run PHPT tests
         working-directory: src/extension/flow-php-ext
-        run: make test
+        run: make test ARROW_SO=../arrow-ext/ext/modules/arrow.so
 
       - name: Install extension and verify
         working-directory: src/extension/flow-php-ext
@@ -104,6 +124,12 @@ jobs:
       - name: Run ETL Integration Tests with Flow PHP Extension
         run: just test --testsuite=etl-integration
 
+      - name: Run CSV Adapter Unit Tests with Flow PHP Extension
+        run: just test --testsuite=adapter-csv-unit
+
+      - name: Run CSV Adapter Integration Tests with Flow PHP Extension
+        run: just test --testsuite=adapter-csv-integration
+
       - name: Upload to Codecov
         if: ${{ !cancelled() && matrix.php == '8.3' && runner.os == 'Linux' }}
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
@@ -138,7 +164,7 @@ jobs:
           tools: 'composer:v2, phpize, php-config'
 
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
+        uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable
         with:
           toolchain: stable
 
@@ -203,7 +229,13 @@ jobs:
 
       - name: Verify extension version
         run: |
-          expected="$(git describe --tags --match '[0-9]*' | sed -E 's/-([0-9]+)-(g[0-9a-f]+)$/+\1.\2/')"
+          describe="$(git describe --tags --match '[0-9]*')"
+          if git describe --exact-match --tags --match '[0-9]*' HEAD >/dev/null 2>&1; then
+            expected="$describe"
+          else
+            # commits past a tag are a pre-release of the next minor: 0.44.1-65-g4525c4adc -> 0.45.0-dev+65.g4525c4adc
+            expected="$(echo "$describe" | awk -F- '{split($1, v, "."); printf "%d.%d.0-dev+%s.%s", v[1], v[2] + 1, $2, $3}')"
+          fi
           actual="$(php -r 'echo phpversion("flow_php");')"
           echo "expected: $expected, actual: $actual"
           test "$actual" = "$expected"

.github/workflows/job-phpunit-telemetry-tests.yml

@@ -54,7 +54,9 @@ jobs:
           extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, curl, pgsql, grpc, protobuf'
           ini-values: 'memory_limit=-1'
           apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev"
-          pie-extensions: "flow-php/pg-query-ext:1.x-dev"
+
+      - name: "Build pg-query-ext from source"
+        uses: "./.github/actions/pg-query-ext"
 
       - name: "Install PHPUnit ${{ matrix.phpunit-version }}"
         if: ${{ matrix.phpunit-version != '11' }}

.github/workflows/job-tests.yml

@@ -113,7 +113,9 @@ jobs:
           extensions: ':psr, apcu, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, curl, pgsql, grpc, protobuf'
           ini-values: 'memory_limit=-1, post_max_size=32M, upload_max_filesize=32M, apc.enable_cli=1'
           apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev"
-          pie-extensions: "flow-php/pg-query-ext:1.x-dev"
+
+      - name: "Build pg-query-ext from source"
+        uses: "./.github/actions/pg-query-ext"
 
       - name: Set up Node.js
         uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0

.github/workflows/pr-check.yml

@@ -18,7 +18,7 @@ jobs:
     permissions:
       pull-requests: write
     steps:
-      - uses: codelytv/pr-size-labeler@095a41fca88b8764fd9e008ad269bcdb82bb38b9 # v1
+      - uses: codelytv/pr-size-labeler@095a41fca88b8764fd9e008ad269bcdb82bb38b9 # v1.10.4
         with:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           xs_label: 'size: XS'

.github/workflows/release.yml

@@ -35,7 +35,9 @@ jobs:
           extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib'
           cache-key-suffix: "-locked-phar"
           apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev"
-          pie-extensions: "flow-php/pg-query-ext:1.x-dev"
+
+      - name: "Build pg-query-ext from source"
+        uses: "./.github/actions/pg-query-ext"
 
       - name: "Build PHAR file"
         run: "just phar"

.github/workflows/test-suite.yml

@@ -11,6 +11,7 @@ on:
       - 'composer.lock'
       - 'composer.json'
       - 'phpunit.xml.dist'
+      - '.nix/pkgs/**'
   push:
     branches: [ 1.x ]
     paths:
@@ -22,6 +23,7 @@ on:
       - 'composer.lock'
       - 'composer.json'
       - 'phpunit.xml.dist'
+      - '.nix/pkgs/**'
   schedule:
     - cron: '0 4 * * *'
   workflow_dispatch:

.github/workflows/test-website.yml

@@ -38,7 +38,9 @@ jobs:
           extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib'
           cache-key-suffix: "-locked-phar"
           apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev"
-          pie-extensions: "flow-php/pg-query-ext:1.x-dev"
+
+      - name: "Build pg-query-ext from source"
+        uses: "./.github/actions/pg-query-ext"
 
       - name: "Build PHAR file"
         run: "just phar"

.nix/blackfire/.gitignore

@@ -0,0 +1 @@
+blackfire.env

.nix/blackfire/blackfire.env.dist

@@ -0,0 +1,2 @@
+# Copy to blackfire.env (gitignored) and fill in; nix-shell exports it for `blackfire run`.
+BLACKFIRE_ENV=

.nix/pkgs/php-arrow-ext/package.nix

@@ -5,7 +5,7 @@
   rustPlatform,
   clang,
   llvmPackages,
-  arrow-ext-version ? "dev",
+  arrow-ext-version ? "0.45.0-dev",
 }:
 
 let
@@ -58,7 +58,7 @@ let
     doCheck = false;
 
     meta = with lib; {
-      description = "Apache Arrow PHP extension powered by Rust";
+      description = "Apache Arrow and Parquet bindings for flow-php/parquet (Rust)";
       license = licenses.mit;
     };
   };

.nix/pkgs/php-flow-php-ext/package.nix

@@ -5,7 +5,7 @@
   rustPlatform,
   clang,
   llvmPackages,
-  flow-php-ext-version ? "dev",
+  flow-php-ext-version ? "0.45.0-dev",
 }:
 
 let
@@ -58,7 +58,7 @@ let
     doCheck = false;
 
     meta = with lib; {
-      description = "Flow PHP native extension (Rust) - Floe frame-body encoder/decoder for DataFrame Rows";
+      description = "Flow PHP native extension (Rust) - native column backend, CSV/JSON readers, Parquet ETL through arrow-ext";
       license = licenses.mit;
     };
   };

benchmarks/src/Datasets/SellersDataset.php

@@ -31,9 +31,8 @@ public function parquet(): string
         $sellerIds = [];
 
         foreach (data_frame()->read(from_parquet($ordersPath)->withColumns(['seller_id']))->get() as $batch) {
-            foreach ($batch->all() as $row) {
-                $sellerId = $row->get('seller_id');
-
+            // @mago-ignore analysis:mixed-assignment
+            foreach ($batch->column('seller_id')->values() as $sellerId) {
                 if (is_scalar($sellerId) || $sellerId instanceof Stringable) {
                     $sellerIds[(string) $sellerId] = true;
                 }

benchmarks/src/Datasets/TextDataset.php

@@ -31,9 +31,12 @@ public function path(): string
         $lines = [];
 
         foreach (data_frame()->read(from_parquet($parquetPath)->withColumns(['customer', 'notes']))->get() as $batch) {
-            foreach ($batch->all() as $row) {
-                $customer = $row->get('customer');
-                $notes = $row->get('notes');
+            $allNotes = $batch->column('notes')->values();
+
+            // @mago-ignore analysis:mixed-assignment
+            foreach ($batch->column('customer')->values() as $i => $customer) {
+                // @mago-ignore analysis:mixed-assignment
+                $notes = $allNotes[$i];
 
                 $customerText = is_scalar($customer) || $customer instanceof Stringable ? (string) $customer : '';
                 $notesText = is_array($notes)

benchmarks/src/Format/Floe/FloeReadScenario.php

@@ -5,28 +5,18 @@
 namespace Flow\Benchmarks\Format\Floe;
 
 use Flow\Benchmarks\Datasets\Datasets;
-use Flow\ETL\Row\PhpRowHydrator;
-use Flow\Floe\FloeEngine;
 
-use function Flow\ETL\DSL\config_builder;
 use function Flow\ETL\DSL\data_frame;
 use function Flow\Floe\DSL\from_floe;
 
 final readonly class FloeReadScenario
 {
     public function __construct(
         private int $rows,
-        private FloeEngine $engine,
     ) {}
 
     public function run(): void
     {
-        $config = config_builder();
-
-        if ($this->engine === FloeEngine::php) {
-            $config->hydrator(new PhpRowHydrator());
-        }
-
-        data_frame($config)->read(from_floe(Datasets::orders($this->rows)->floe(), engine: $this->engine))->run();
+        data_frame()->read(from_floe(Datasets::orders($this->rows)->floe()))->run();
     }
 }